diff --git a/.gitignore b/.gitignore index 24b82fc3..150023ac 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ backend.tf backend-config.hcl credentials.json key.json +terraform-ls.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c99c53..365956b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,70 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- fix external IP assignment in `compute-vm` + +## [2.3.0] - 2020-07-02 + +- new 'Cloud Storage to Bigquery with Cloud Dataflow' end to end data solution +- **incompatible change** additive IAM bindings are now keyed by identity instead of role, and use a single `iam_additive_bindings` variable, refer to [#103] for details +- set `delete_contents_on_destroy` in the foundations examples audit dataset to allow destroying +- trap errors raised by the `project` module on destroy + +## [2.2.0] - 2020-06-29 + +- make project creation optional in `project` module to allow managing a pre-existing project +- new `cloud-endpoints` module +- new `cloud-function` module + +## [2.1.0] - 2020-06-22 + +- **incompatible change** routes in the `net-vpc` module now interpolate the VPC name to ensure uniqueness, upgrading from a previous version will drop and recreate routes +- the top-level `docker-images` folder has been moved inside `modules/cloud-config-container/onprem` +- `dns_keys` output added to the `dns` module +- add `group-config` variable, `groups` and `group_self_links` outputs to `net-ilb` module to allow creating ILBs for externally managed instances +- make the IAM bindings depend on the compute instance in the `compute-vm` module + +## [2.0.0] - 2020-06-11 + +- new `data-solutions` section and `cmek-via-centralized-kms` example +- **incompatible change** static VPN routes now interpolate the VPN gateway name to enforce uniqueness, upgrading from a previous version will drop and recreate routes + +## [1.9.0] - 2020-06-10 + +- new `bigtable-instance` module +- add support for IAM bindings to `compute-vm` module + +## [1.8.1] - 2020-06-07 + +- use `all` instead of specifying protocols in the admin firewall rule of the `net-vpc-firewall` module +- add support for encryption keys in `gcs` module +- set `next_hop_instance_zone` in `net-vpc` for next hop instance routes to avoid triggering recreation + +## [1.8.0] - 2020-06-03 + +- **incompatible change** the `kms` module has been refactored and will be incompatible with previous state +- **incompatible change** robot and default service accounts outputs in the `project` module have been refactored and are now exposed via a single `service_account` output (cf [#82]) +- add support for PD CSI driver in GKE module +- refactor `iam-service-accounts` module outputs to be more resilient +- add option to use private GCR to `cos-generic-metadata` module + +## [1.7.0] - 2020-05-30 + +- add support for disk encryption to the `compute-vm` module +- new `datafusion` module +- new `container-registry` module +- new `artifact-registry` module + +## [1.6.0] - 2020-05-20 + +- add output to `gke-cluster` exposing the cluster's CA certificate +- fix `gke-cluster` autoscaling options +- add support for Service Directory bound zones to the `dns` module +- new `service-directory` module +- new `source-repository` module + +## [1.5.0] - 2020-05-11 + - **incompatible change** the `bigquery` module has been removed and replaced by the new `bigquery-dataset` module - **incompatible change** subnets in the `net-vpc` modules are now passed as a list instead of map, and all related variables for IAM and flow logs use `region/name` instead of `name` keys; it's now possible to have the same subnet name in different regions - replace all references to the removed `resourceviews.googleapis.com` API with `container.googleapis.com` @@ -11,7 +75,7 @@ All notable changes to this project will be documented in this file. - fix health checks in `compute-mig` and `net-ilb` modules - new `cos-generic-metadata` module in the `cloud-config-container` suite - new `envoy-traffic-director` module in the `cloud-config-container` suite -- new `pubsub` module (untested) +- new `pubsub` module ## [1.4.1] - 2020-05-02 @@ -55,10 +119,22 @@ All notable changes to this project will be documented in this file. - merge development branch with suite of new modules and end-to-end examples -[Unreleased]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.4.1...HEAD +[Unreleased]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.3.0...HEAD +[2.3.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.2.0...v2.3.0 +[2.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.1.0...v2.2.0 +[2.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v2.0.0...v2.1.0 +[2.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.9.0...v2.0.0 +[1.9.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.8.1...v1.9.0 +[1.8.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.8.0...v1.8.1 +[1.8.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.7.0...v1.8.0 +[1.7.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.6.0...v1.7.0 +[1.6.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.5.0...v1.6.0 +[1.5.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.4.1...v1.5.0 [1.4.1]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.4.0...v1.4.1 [1.4.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.3.0...v1.4.0 [1.3.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.2...v1.3.0 [1.2.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.1...v1.2 [1.1.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v1.0...v1.1 [1.0.0]: https://github.com/terraform-google-modules/cloud-foundation-fabric/compare/v0.1...v1.0 +[#82]: https://github.com/terraform-google-modules/cloud-foundation-fabric/pull/82 +[#103]: https://github.com/terraform-google-modules/cloud-foundation-fabric/pull/103 diff --git a/README.md b/README.md index 30c51719..8ca9bca1 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,9 @@ Currently available examples: - **foundations** - [single level hierarchy](./foundations/environments/) (environments), [multiple level hierarchy](./foundations/business-units/) (business units + environments) - **infrastructure** - [hub and spoke via peering](./infrastructure/hub-and-spoke-peering/), [hub and spoke via VPN](./infrastructure/hub-and-spoke-vpn/), [DNS and Google Private Access for on-premises](./infrastructure/onprem-google-access-dns/), [Shared VPC with GKE support](./infrastructure/shared-vpc-gke/) +- **data solutions** - [GCE/GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms/), [Cloud Storage to Bigquery with Cloud Dataflow](./data-solutions/gcs-to-bq-with-dataflow/) -For more information see the README files in the [foundations](./foundations/) and [infrastructure](./infrastructure/) folders. +For more information see the README files in the [foundations](./foundations/), [infrastructure](./infrastructure/) and [data solutions](./data-solutions/) folders. ## Modules @@ -33,9 +34,11 @@ The current list of modules supports most of the core foundational and networkin Currently available modules: - **foundational** - [folders](./modules/folders), [log sinks](./modules/logging-sinks), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-accounts) -- **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb) +- **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb), [Service Directory](./modules/service-directory), [Cloud Endpoints](./modules/cloudenpoints) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cos-container) (coredns, mysql, onprem, squid) -- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset) +- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance) +- **development** - [Cloud Source Repository](./modules/source-repository), [Container Registry](./modules/container-registry), [Artifact Registry](./modules/artifact-registry) - **security** - [KMS](./modules/kms), [SecretManager](./modules/secret-manager) +- **serverless** - [Cloud Functions](./cloud-functions) For more information and usage examples see each module's README file. diff --git a/data-solutions/README.md b/data-solutions/README.md new file mode 100644 index 00000000..18e43825 --- /dev/null +++ b/data-solutions/README.md @@ -0,0 +1,16 @@ +# GCP Data Services examples + +The examples in this folder implement **typical data service topologies** and **end-to-end scenarios**, that allow testing specific features like Cloud KMS to encrypt your data, or VPC-SC to mitigate data exfiltration. + +They are meant to be used as minimal but complete starting points to create actual infrastructure, and as playgrounds to experiment with specific Google Cloud features. + +## Examples + +### GCE and GCS CMEK via centralized Cloud KMS + + This [example](./cmek-via-centralized-kms/) implements [CMEK](https://cloud.google.com/kms/docs/cmek) for GCS and GCE, via keys hosted in KMS running in a centralized project. The example shows the basic resources and permissions for the typical use case of application projects implementing encryption at rest via a centrally managed KMS service. +
+ +### Cloud Storage to Bigquery with Cloud Dataflow + This [example](./gcs-to-bq-with-dataflow/) implements [Cloud Storage](https://cloud.google.com/kms/docs/cmek) to Bigquery data import using Cloud Dataflow. +All resources use CMEK hosted in Cloud KMS running in a centralized project. The example shows the basic resources and permissions for the typical use case to read, transform and import data from Cloud Storage to Bigquery. diff --git a/data-solutions/cmek-via-centralized-kms/README.md b/data-solutions/cmek-via-centralized-kms/README.md new file mode 100644 index 00000000..62b42210 --- /dev/null +++ b/data-solutions/cmek-via-centralized-kms/README.md @@ -0,0 +1,58 @@ +# GCE and GCS CMEK via centralized Cloud KMS + +This example creates a sample centralized [Cloud KMS](https://cloud.google.com/kms?hl=it) configuration, and uses it to implement CMEK for [Cloud Storage](https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys) and [Compute Engine](https://cloud.google.com/compute/docs/disks/customer-managed-encryption) in a separate project. + +The example is designed to match real-world use cases with a minimum amount of resources, and be used as a starting point for scenarios where application projects implement CMEK using keys managed by a central team. It also includes the IAM wiring needed to make such scenarios work. + +This is the high level diagram: + +![High-level diagram](diagram.png "High-level diagram") + +## Managed resources and services + +This sample creates several distinct groups of resources: + +- projects + - Cloud KMS project + - Service Project configured for GCE instances and GCS buckets +- networking + - VPC network + - One subnet + - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC +- IAM + - One service account for the GGE instance +- KMS + - One key ring + - One crypto key (Procection level: softwere) for Cloud Engine + - One crypto key (Protection level: softwere) for Cloud Storage +- GCE + - One instance encrypted with a CMEK Cryptokey hosted in Cloud KMS +- GCS + - One bucket encrypted with a CMEK Cryptokey hosted in Cloud KMS + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account | Billing account id used as default for new projects. | string | ✓ | | +| root_node | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | | +| *location* | The location where resources will be deployed. | string | | europe | +| *project_kms_name* | Name for the new KMS Project. | string | | my-project-kms-001 | +| *project_service_name* | Name for the new Service Project. | string | | my-project-service-001 | +| *region* | The region where resources will be deployed. | string | | europe-west1 | +| *vpc_ip_cidr_range* | Ip range used in the subnet deployef in the Service Project. | string | | 10.0.0.0/20 | +| *vpc_name* | Name of the VPC created in the Service Project. | string | | local | +| *vpc_subnet_name* | Name of the subnet created in the Service Project. | string | | subnet | +| *zone* | The zone where resources will be deployed. | string | | europe-west1-b | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| bucket | GCS Bucket Cloud KMS crypto keys. | | +| bucket_keys | GCS Bucket Cloud KMS crypto keys. | | +| projects | Project ids. | | +| vm | GCE VMs. | | +| vm_keys | GCE VM Cloud KMS crypto keys. | | + diff --git a/data-solutions/cmek-via-centralized-kms/backend.tf.sample b/data-solutions/cmek-via-centralized-kms/backend.tf.sample new file mode 100644 index 00000000..a540c7cd --- /dev/null +++ b/data-solutions/cmek-via-centralized-kms/backend.tf.sample @@ -0,0 +1,20 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +terraform { + backend "gcs" { + bucket = "" + } +} diff --git a/data-solutions/cmek-via-centralized-kms/diagram.png b/data-solutions/cmek-via-centralized-kms/diagram.png new file mode 100644 index 00000000..18fe1adf Binary files /dev/null and b/data-solutions/cmek-via-centralized-kms/diagram.png differ diff --git a/data-solutions/cmek-via-centralized-kms/main.tf b/data-solutions/cmek-via-centralized-kms/main.tf new file mode 100644 index 00000000..464f208f --- /dev/null +++ b/data-solutions/cmek-via-centralized-kms/main.tf @@ -0,0 +1,155 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Projects # +############################################################################### + +module "project-service" { + source = "../../modules/project" + name = var.project_service_name + parent = var.root_node + billing_account = var.billing_account + services = [ + "compute.googleapis.com", + "servicenetworking.googleapis.com", + "storage-component.googleapis.com" + ] + oslogin = true +} + +module "project-kms" { + source = "../../modules/project" + name = var.project_kms_name + parent = var.root_node + billing_account = var.billing_account + services = [ + "cloudkms.googleapis.com", + "servicenetworking.googleapis.com" + ] + oslogin = true +} + +############################################################################### +# Networking # +############################################################################### + +module "vpc" { + source = "../../modules/net-vpc" + project_id = module.project-service.project_id + name = var.vpc_name + subnets = [ + { + ip_cidr_range = var.vpc_ip_cidr_range + name = var.vpc_subnet_name + region = var.region + secondary_ip_range = {} + } + ] +} + +module "vpc-firewall" { + source = "../../modules/net-vpc-firewall" + project_id = module.project-service.project_id + network = module.vpc.name + admin_ranges_enabled = true + admin_ranges = [var.vpc_ip_cidr_range] +} + +############################################################################### +# KMS # +############################################################################### + +module "kms" { + source = "../../modules/kms" + project_id = module.project-kms.project_id + keyring = { + name = "my-keyring", + location = var.location + } + keys = { key-gce = null, key-gcs = null } + key_iam_roles = { + key-gce = ["roles/cloudkms.cryptoKeyEncrypterDecrypter"] + } + key_iam_members = { + key-gce = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + "serviceAccount:${module.project-service.service_accounts.robots.compute}", + ] + }, + key-gcs = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + "serviceAccount:${module.project-service.service_accounts.robots.storage}", + ] + } + } +} + +############################################################################### +# GCE # +############################################################################### + +module "kms_vm_example" { + source = "../../modules/compute-vm" + project_id = module.project-service.project_id + region = var.region + zone = var.zone + name = "kms-vm" + network_interfaces = [{ + network = module.vpc.self_link, + subnetwork = module.vpc.subnet_self_links["${var.region}/subnet"], + nat = false, + addresses = null + }] + attached_disks = [ + { + name = "attacheddisk" + size = 10 + image = null + options = { + auto_delete = true + mode = null + source = null + type = null + } + } + ] + instance_count = 1 + boot_disk = { + image = "projects/debian-cloud/global/images/family/debian-10" + type = "pd-ssd" + size = 10 + encrypt_disk = true + } + tags = ["ssh"] + encryption = { + encrypt_boot = true + disk_encryption_key_raw = null + kms_key_self_link = module.kms.key_self_links.key-gce + } +} + +############################################################################### +# GCS # +############################################################################### + +module "kms-gcs" { + source = "../../modules/gcs" + project_id = module.project-service.project_id + prefix = "my-bucket-001" + names = ["kms-gcs"] + encryption_keys = { + kms-gcs = module.kms.keys.key-gce.self_link, + } +} diff --git a/data-solutions/cmek-via-centralized-kms/outputs.tf b/data-solutions/cmek-via-centralized-kms/outputs.tf new file mode 100644 index 00000000..99d26e15 --- /dev/null +++ b/data-solutions/cmek-via-centralized-kms/outputs.tf @@ -0,0 +1,53 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "bucket" { + description = "GCS Bucket Cloud KMS crypto keys." + value = { + for bucket in module.kms-gcs.buckets : + bucket.name => bucket.url + } +} + +output "bucket_keys" { + description = "GCS Bucket Cloud KMS crypto keys." + value = { + for bucket in module.kms-gcs.buckets : + bucket.name => bucket.encryption + } +} + +output "projects" { + description = "Project ids." + value = { + service-project = module.project-service.project_id + kms-project = module.project-kms.project_id + } +} + +output "vm" { + description = "GCE VMs." + value = { + for instance in module.kms_vm_example.instances : + instance.name => instance.network_interface.0.network_ip + } +} + +output "vm_keys" { + description = "GCE VM Cloud KMS crypto keys." + value = { + for instance in module.kms_vm_example.instances : + instance.name => instance.boot_disk.0.kms_key_self_link + } +} diff --git a/data-solutions/cmek-via-centralized-kms/variables.tf b/data-solutions/cmek-via-centralized-kms/variables.tf new file mode 100644 index 00000000..fdc0ac94 --- /dev/null +++ b/data-solutions/cmek-via-centralized-kms/variables.tf @@ -0,0 +1,72 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "billing_account" { + description = "Billing account id used as default for new projects." + type = string +} + +variable "location" { + description = "The location where resources will be deployed." + type = string + default = "europe" +} + +variable "project_service_name" { + description = "Name for the new Service Project." + type = string + default = "my-project-service-001" +} + +variable "project_kms_name" { + description = "Name for the new KMS Project." + type = string + default = "my-project-kms-001" +} + +variable "region" { + description = "The region where resources will be deployed." + type = string + default = "europe-west1" +} + +variable "root_node" { + description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id." + type = string +} + +variable "vpc_name" { + description = "Name of the VPC created in the Service Project." + type = string + default = "local" +} + +variable "vpc_subnet_name" { + description = "Name of the subnet created in the Service Project." + type = string + default = "subnet" +} + +variable "vpc_ip_cidr_range" { + description = "Ip range used in the subnet deployef in the Service Project." + type = string + default = "10.0.0.0/20" +} + +variable "zone" { + description = "The zone where resources will be deployed." + type = string + default = "europe-west1-b" +} diff --git a/data-solutions/cmek-via-centralized-kms/versions.tf b/data-solutions/cmek-via-centralized-kms/versions.tf new file mode 100644 index 00000000..057095c0 --- /dev/null +++ b/data-solutions/cmek-via-centralized-kms/versions.tf @@ -0,0 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.12.6" +} diff --git a/data-solutions/gcs-to-bq-with-dataflow/README.md b/data-solutions/gcs-to-bq-with-dataflow/README.md new file mode 100644 index 00000000..23a4470d --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/README.md @@ -0,0 +1,138 @@ +# Cloud Storage to Bigquery with Cloud Dataflow + +This example creates the infrastructure needed to run a [Cloud Dataflow](https://cloud.google.com/dataflow) pipeline to import data from [GCS](https://cloud.google.com/storage) to [Bigquery](https://cloud.google.com/bigquery). + +The solution will use: + - internal IPs for GCE and Dataflow instances + - CMEK encription for GCS bucket, GCE instances, DataFlow instances and BigQuery tables + - Cloud NAT to let resources comunicate to the Internet, run system updates, and install packages + +The example is designed to match real-world use cases with a minimum amount of resources. It can be used as a starting point for more complex scenarios. + +This is the high level diagram: + +![GCS to Biquery High-level diagram](diagram.png "GCS to Biquery High-level diagram") + +## Managed resources and services + +This sample creates several distinct groups of resources: + +- projects + - Cloud KMS project + - Service Project configured for GCE instances, GCS buckets, Dataflow instances and BigQuery tables +- networking + - VPC network + - One subnet + - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC +- IAM + - One service account for GGE instances + - One service account for Dataflow instances + - One service account for Bigquery tables +- KMS + - One contintent key ring (example: 'Europe') + - One crypto key (Procection level: softwere) for Cloud Engine + - One crypto key (Protection level: softwere) for Cloud Storage + - One regional key ring ('example: 'europe-west1') + - One crypto key (Protection level: softwere) for Cloud Dataflow +- GCE + - One instance encrypted with a CMEK Cryptokey hosted in Cloud KMS +- GCS + - One bucket encrypted with a CMEK Cryptokey hosted in Cloud KMS +- BQ + - One dataset encrypted with a CMEK Cryptokey hosted in Cloud KMS + - Two tables encrypted with a CMEK Cryptokey hosted in Cloud KMS + +## Test your environment with Cloud Dataflow +You can now connect to the GCE instance with the following command: + +```hcl + gcloud compute ssh vm-example-1 +``` + +You can run now the simple pipeline you can find [here](./script/data_ingestion/). Once you have installed required packages and copied a file into the GCS bucket, you can trigger the pipeline using internal ips with a command simila to: + +```hcl +python data_ingestion.py \ +--runner=DataflowRunner \ +--max_num_workers=10 \ +--autoscaling_algorithm=THROUGHPUT_BASED \ +--region=### REGION ### \ +--staging_location=gs://### TEMP BUCKET NAME ###/ \ +--temp_location=gs://### TEMP BUCKET NAME ###/ \ +--project=### PROJECT ID ### \ +--input=gs://### DATA BUCKET NAME###/### FILE NAME ###.csv \ +--output=### DATASET NAME ###.### TABLE NAME ### \ +--service_account_email=### SERVICE ACCOUNT EMAIL ### \ +--network=### NETWORK NAME ### \ +--subnetwork=### SUBNET NAME ### \ +--dataflow_kms_key=### CRYPTOKEY ID ### \ +--no_use_public_ips +``` + +for example: + +```hcl +python data_ingestion.py \ +--runner=DataflowRunner \ +--max_num_workers=10 \ +--autoscaling_algorithm=THROUGHPUT_BASED \ +--region=europe-west1 \ +--staging_location=gs://lc-001-eu-df-tmplocation/ \ +--temp_location=gs://lc-001-eu-df-tmplocation/ \ +--project=lcaggio-demo \ +--input=gs://lc-eu-data/person.csv \ +--output=bq_dataset.df_import \ +--service_account_email=df-test@lcaggio-demo.iam.gserviceaccount.com \ +--network=local \ +--subnetwork=regions/europe-west1/subnetworks/subnet \ +--dataflow_kms_key=projects/lcaggio-demo-kms/locations/europe-west1/keyRings/my-keyring-regional/cryptoKeys/key-df \ +--no_use_public_ips +``` + +You can check data imported into Google BigQuery from the Google Cloud Console UI. + +## Test your environment with 'bq' CLI +You can now connect to the GCE instance with the following command: + +```hcl + gcloud compute ssh vm-example-1 +``` + +You can run now a simple 'bq load' command to import data into Bigquery. Below an example command: + +```hcl +bq load \ +--source_format=CSV \ +bq_dataset.bq_import \ +gs://my-bucket/person.csv \ +schema_bq_import.json +``` + +You can check data imported into Google BigQuery from the Google Cloud Console UI. + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| billing_account | Billing account id used as default for new projects. | string | ✓ | | +| project_kms_name | Name for the new KMS Project. | string | ✓ | | +| project_service_name | Name for the new Service Project. | string | ✓ | | +| root_node | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | | +| *location* | The location where resources will be deployed. | string | | europe | +| *region* | The region where resources will be deployed. | string | | europe-west1 | +| *ssh_source_ranges* | IP CIDR ranges that will be allowed to connect via SSH to the onprem instance. | list(string) | | ["0.0.0.0/0"] | +| *vpc_ip_cidr_range* | Ip range used in the subnet deployef in the Service Project. | string | | 10.0.0.0/20 | +| *vpc_name* | Name of the VPC created in the Service Project. | string | | local | +| *vpc_subnet_name* | Name of the subnet created in the Service Project. | string | | subnet | +| *zone* | The zone where resources will be deployed. | string | | europe-west1-b | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| bq_tables | Bigquery Tables. | | +| buckets | GCS Bucket Cloud KMS crypto keys. | | +| projects | Project ids. | | +| vm | GCE VMs. | | + diff --git a/data-solutions/gcs-to-bq-with-dataflow/backend.tf.sample b/data-solutions/gcs-to-bq-with-dataflow/backend.tf.sample new file mode 100644 index 00000000..a540c7cd --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/backend.tf.sample @@ -0,0 +1,20 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +terraform { + backend "gcs" { + bucket = "" + } +} diff --git a/data-solutions/gcs-to-bq-with-dataflow/diagram.png b/data-solutions/gcs-to-bq-with-dataflow/diagram.png new file mode 100644 index 00000000..2b1cb198 Binary files /dev/null and b/data-solutions/gcs-to-bq-with-dataflow/diagram.png differ diff --git a/data-solutions/gcs-to-bq-with-dataflow/main.tf b/data-solutions/gcs-to-bq-with-dataflow/main.tf new file mode 100644 index 00000000..84521c03 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/main.tf @@ -0,0 +1,342 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + vm-startup-script = join("\n", [ + "#! /bin/bash", + "apt-get update && apt-get install -y bash-completion git python3-venv gcc build-essential python-dev python3-dev", + "pip3 install --upgrade setuptools pip" + ]) +} + +############################################################################### +# Projects - Centralized # +############################################################################### + +module "project-service" { + source = "../../modules/project" + name = var.project_service_name + parent = var.root_node + billing_account = var.billing_account + services = [ + "compute.googleapis.com", + "servicenetworking.googleapis.com", + "storage-component.googleapis.com", + "bigquery.googleapis.com", + "bigquerystorage.googleapis.com", + "bigqueryreservation.googleapis.com", + "dataflow.googleapis.com", + "cloudkms.googleapis.com", + ] + oslogin = true +} + +module "project-kms" { + source = "../../modules/project" + name = var.project_kms_name + parent = var.root_node + billing_account = var.billing_account + services = [ + "cloudkms.googleapis.com", + ] +} + +############################################################################### +# Project Service Accounts # +############################################################################### + +module "service-account-bq" { + source = "../../modules/iam-service-accounts" + project_id = module.project-service.project_id + names = ["bq-test"] + iam_project_roles = { + (module.project-service.project_id) = [ + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + "roles/bigquery.admin" + ] + } +} + +module "service-account-gce" { + source = "../../modules/iam-service-accounts" + project_id = module.project-service.project_id + names = ["gce-test"] + iam_project_roles = { + (module.project-service.project_id) = [ + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + "roles/dataflow.admin", + "roles/iam.serviceAccountUser", + "roles/bigquery.dataOwner", + "roles/bigquery.jobUser" # Needed to import data using 'bq' command + ] + } +} + +module "service-account-df" { + source = "../../modules/iam-service-accounts" + project_id = module.project-service.project_id + names = ["df-test"] + iam_project_roles = { + (module.project-service.project_id) = [ + "roles/dataflow.worker", + "roles/bigquery.dataOwner", + "roles/bigquery.metadataViewer", + "roles/storage.objectViewer", + "roles/bigquery.jobUser" + ] + } +} + +data "google_bigquery_default_service_account" "bq_sa" { + project = module.project-service.project_id +} + +data "google_storage_project_service_account" "gcs_account" { + project = module.project-service.project_id +} + +############################################################################### +# KMS # +############################################################################### + +module "kms" { + source = "../../modules/kms" + project_id = module.project-kms.project_id + keyring = { + name = "my-keyring", + location = var.location + } + keys = { key-gce = null, key-gcs = null, key-bq = null } + key_iam_roles = { + key-gce = ["roles/cloudkms.cryptoKeyEncrypterDecrypter"] + key-gcs = ["roles/cloudkms.cryptoKeyEncrypterDecrypter"] + key-bq = ["roles/cloudkms.cryptoKeyEncrypterDecrypter"] + } + key_iam_members = { + key-gce = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + "serviceAccount:${module.project-service.service_accounts.robots.compute}", + ] + }, + key-gcs = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + #"serviceAccount:${module.project-service.service_accounts.robots.storage}", + "serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}" + ] + }, + key-bq = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + # TODO: Find a better place to store BQ service account + #"serviceAccount:${module.project-service.service_accounts.default.bq}", + "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}", + ] + }, + } +} + +module "kms-regional" { + source = "../../modules/kms" + project_id = module.project-kms.project_id + keyring = { + name = "my-keyring-regional", + location = var.region + } + keys = { key-df = null } + key_iam_roles = { + key-df = ["roles/cloudkms.cryptoKeyEncrypterDecrypter"] + } + key_iam_members = { + key-df = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + "serviceAccount:${module.project-service.service_accounts.robots.dataflow}", + "serviceAccount:${module.project-service.service_accounts.robots.compute}", + ] + } + } +} + +############################################################################### +# Networking # +############################################################################### + +module "vpc" { + source = "../../modules/net-vpc" + project_id = module.project-service.project_id + name = var.vpc_name + subnets = [ + { + ip_cidr_range = var.vpc_ip_cidr_range + name = var.vpc_subnet_name + region = var.region + secondary_ip_range = {} + } + ] +} + +module "vpc-firewall" { + source = "../../modules/net-vpc-firewall" + project_id = module.project-service.project_id + network = module.vpc.name + admin_ranges_enabled = true + admin_ranges = [var.vpc_ip_cidr_range] +} + +module "nat" { + source = "../../modules/net-cloudnat" + project_id = module.project-service.project_id + region = var.region + name = "default" + router_network = module.vpc.name +} + +############################################################################### +# GCE # +############################################################################### + +module "vm_example" { + source = "../../modules/compute-vm" + project_id = module.project-service.project_id + region = var.region + zone = var.zone + name = "vm-example" + network_interfaces = [{ + network = module.vpc.self_link, + subnetwork = module.vpc.subnet_self_links["${var.region}/${var.vpc_subnet_name}"], + nat = false, + addresses = null + }] + attached_disks = [ + { + name = "attacheddisk" + size = 10 + image = null + options = { + auto_delete = true + mode = null + source = null + type = null + } + } + ] + instance_count = 2 + boot_disk = { + image = "projects/debian-cloud/global/images/family/debian-10" + type = "pd-ssd" + size = 10 + encrypt_disk = true + } + encryption = { + encrypt_boot = true + disk_encryption_key_raw = null + kms_key_self_link = module.kms.key_self_links.key-gce + } + metadata = { startup-script = local.vm-startup-script } + service_account = module.service-account-gce.email + service_account_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + tags = ["ssh"] +} + +############################################################################### +# GCS # +############################################################################### + +module "kms-gcs" { + source = "../../modules/gcs" + project_id = module.project-service.project_id + prefix = module.project-service.project_id + names = ["data", "df-tmplocation"] + iam_roles = { + data = ["roles/storage.admin","roles/storage.objectViewer"], + df-tmplocation = ["roles/storage.admin"] + } + iam_members = { + data = { + "roles/storage.admin" = [ + "serviceAccount:${module.service-account-gce.email}", + ], + "roles/storage.viewer" = [ + "serviceAccount:${module.service-account-df.email}", + ], + }, + df-tmplocation = { + "roles/storage.admin" = [ + "serviceAccount:${module.service-account-gce.email}", + "serviceAccount:${module.service-account-df.email}", + ] + } + } + encryption_keys = { + data = module.kms.keys.key-gcs.self_link, + df-tmplocation = module.kms.keys.key-gcs.self_link, + } + force_destroy = { + data = true, + df-tmplocation = true, + } +} + +############################################################################### +# BQ # +############################################################################### + +module "bigquery-dataset" { + source = "../../modules/bigquery-dataset" + project_id = module.project-service.project_id + id = "bq_dataset" + access_roles = { + reader-group = { role = "READER", type = "domain" } + owner = { role = "OWNER", type = "user_by_email" } + } + access_identities = { + reader-group = "caggioland.com" + owner = module.service-account-bq.email + } + encryption_key = module.kms.keys.key-bq.self_link + tables = { + bq_import = { + friendly_name = "BQ import" + labels = {} + options = null + partitioning = { + field = null + range = null # use start/end/interval for range + time = null + } + schema = file("schema_bq_import.json") + options = { + clustering = null + expiration_time = null + encryption_key = module.kms.keys.key-bq.self_link + } + }, + df_import = { + friendly_name = "Dataflow import" + labels = {} + options = null + partitioning = { + field = null + range = null # use start/end/interval for range + time = null + } + schema = file("schema_df_import.json") + options = { + clustering = null + expiration_time = null + encryption_key = module.kms.keys.key-bq.self_link + } + } + } +} diff --git a/data-solutions/gcs-to-bq-with-dataflow/outputs.tf b/data-solutions/gcs-to-bq-with-dataflow/outputs.tf new file mode 100644 index 00000000..9dc4de31 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/outputs.tf @@ -0,0 +1,42 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "bq_tables" { + description = "Bigquery Tables." + value = module.bigquery-dataset.table_ids +} + +output "buckets" { + description = "GCS Bucket Cloud KMS crypto keys." + value = { + for bucket in module.kms-gcs.buckets : + bucket.name => bucket.url + } +} + +output "projects" { + description = "Project ids." + value = { + service-project = module.project-service.project_id + kms-project = module.project-kms.project_id + } +} + +output "vm" { + description = "GCE VMs." + value = { + for instance in module.vm_example.instances : + instance.name => instance.network_interface.0.network_ip + } +} diff --git a/data-solutions/gcs-to-bq-with-dataflow/schema_bq_import.json b/data-solutions/gcs-to-bq-with-dataflow/schema_bq_import.json new file mode 100644 index 00000000..e26fa3d2 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/schema_bq_import.json @@ -0,0 +1,14 @@ +[ + { + "name": "name", + "type": "STRING" + }, + { + "name": "surname", + "type": "STRING" + }, + { + "name": "age", + "type": "NUMERIC" + } +] \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/schema_df_import.json b/data-solutions/gcs-to-bq-with-dataflow/schema_df_import.json new file mode 100644 index 00000000..00e428d7 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/schema_df_import.json @@ -0,0 +1,22 @@ +[ + { + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "surname", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "age", + "type": "NUMERIC" + }, + { + "mode": "NULLABLE", + "name": "_TIMESTAMP", + "type": "TIMESTAMP" + } +] \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/README.md b/data-solutions/gcs-to-bq-with-dataflow/scripts/README.md new file mode 100644 index 00000000..2ab41357 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/README.md @@ -0,0 +1,4 @@ +# Sripts +In this section you can find two simple scripts to test your environment: + - [Data ingestion](./data_ingestion/): a simple Apache Beam Python pipeline to import data from Google Cloud Storage into Bigquery. + - [Person details generator](./person_details_generator/): a simple script to generate some random data to test your environment. \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/LICENSE b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/README.md b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/README.md new file mode 100644 index 00000000..badc4d1c --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/README.md @@ -0,0 +1,99 @@ +# Ingest CSV files from GCS into Bigquery + +In this example we create a Python [Apache Beam](https://beam.apache.org/) pipeline running on [Google Cloud Dataflow](https://cloud.google.com/dataflow/) to import CSV files into BigQuery adding a timestamp to each row. Below the architecture used: + +![Apache Beam pipeline to import CSV from GCS into BQ](diagram.png) + +The architecture uses: +* [Google Cloud Storage]() to store CSV source files +* [Google Cloud Dataflow](https://cloud.google.com/dataflow/) to read files from Google Cloud Storage, Transform data base on the structure of the file and import the data into Google BigQuery +* [Google BigQuery](https://cloud.google.com/bigquery/) to store data in a Data Lake. + +You can use this script as a starting point to import your files into Google BigQuery. You'll probably need to adapt the script logic to your requirements. + +## 1. Prerequisites + - Up and running GCP project with enabled billing account + - gcloud installed and initiated to your project + - Google Cloud Dataflow API enabled + - Google Cloud Storage Bucket containing the file to import (CSV format) containings name, surnames and age. Example: `Mario,Rossi,30`. + - Google Cloud Storage Bucket for temp and staging Google Dataflow files + - Google BigQuery dataset + - [Python](https://www.python.org/) >= 3.7 and python-dev module + - gcc + - Google Cloud [Application Default Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login) + +## 2. Create virtual environment +Create a new virtual environment (recommended) and install requirements: + +``` +virtualenv env +source ./env/bin/activate +pip3 install --upgrade setuptools pip +pip3 install -r requirements.txt +``` + +## 4. Upload files into Google Cloud Storage +Upload files to be imported into Google Bigquery in a Google Cloud Storage Bucket. You can use `gsutil` using a command like: +``` +gsutil cp [LOCAL_OBJECT_LOCATION] gs://[DESTINATION_BUCKET_NAME]/ +``` + +Files need to be in CSV format,For example: +``` +Enrico,Bianchi,20 +Mario,Rossi,30 +``` + +You can use the [person_details_generator](../person_details_generator/) script if you want to create random person details. + +## 5. Run pipeline +You can check parameters accepted by the `data_ingestion.py` script with the following command: +``` +python pipelines/data_ingestion --help +``` + +You can run the pipeline locally with the following command: +``` +python data_ingestion.py \ +--runner=DirectRunner \ +--project=###PUT HERE PROJECT ID### \ +--input=###PUT HERE THE FILE TO IMPORT. EXAMPLE: gs://bucket_name/person.csv ### \ +--output=###PUT HERE BQ DATASET.TABLE### +``` + +or you can run the pipeline on Google Dataflow using the following command: + +``` +python data_ingestion.py \ +--runner=DataflowRunner \ +--max_num_workers=100 \ +--autoscaling_algorithm=THROUGHPUT_BASED \ +--region=###PUT HERE REGION### \ +--staging_location=###PUT HERE GCS STAGING LOCATION### \ +--temp_location=###PUT HERE GCS TMP LOCATION###\ +--project=###PUT HERE PROJECT ID### \ +--input=###PUT HERE GCS BUCKET NAME. EXAMPLE: gs://bucket_name/person.csv### \ +--output=###PUT HERE BQ DATASET NAME. EXAMPLE: bq_dataset.df_import### \ +``` + +Below an example to run the pipeline specifying Network and Subnetwork, using private IPs and using a KMS key to encrypt data at rest: + +``` +python data_ingestion.py \ +--runner=DataflowRunner \ +--max_num_workers=100 \ +--autoscaling_algorithm=THROUGHPUT_BASED \ +--region=###PUT HERE REGION### \ +--staging_location=###PUT HERE GCS STAGING LOCATION### \ +--temp_location=###PUT HERE GCS TMP LOCATION###\ +--project=###PUT HERE PROJECT ID### \ +--network=###PUT HERE YOUR NETWORK### \ +--subnetwork=###PUT HERE YOUR SUBNETWORK. EXAMPLE: regions/europe-west1/subnetworks/subnet### \ +--dataflowKmsKey=###PUT HERE KMES KEY. Example: projects/lcaggio-d-4-kms/locations/europe-west1/keyRings/my-keyring-regional/cryptoKeys/key-df### \ +--input=###PUT HERE GCS BUCKET NAME. EXAMPLE: gs://bucket_name/person.csv### \ +--output=###PUT HERE BQ DATASET NAME. EXAMPLE: bq_dataset.df_import### \ +--no_use_public_ips +``` + +## 6. Check results +You can check data imported into Google BigQuery from the Google Cloud Console UI. \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/REQUIREMENTS.txt b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/REQUIREMENTS.txt new file mode 100644 index 00000000..32bfbbd1 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/REQUIREMENTS.txt @@ -0,0 +1,3 @@ +apache-beam[gcp] +setuptools +wheel \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/data_ingestion.py b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/data_ingestion.py new file mode 100644 index 00000000..643cdc87 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/data_ingestion.py @@ -0,0 +1,134 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataflow pipeline. Reads a CSV file and writes to a BQ table adding a timestamp. +""" + + +import argparse +import logging +import re + +import apache_beam as beam +from apache_beam.options.pipeline_options import PipelineOptions + + +class DataIngestion: + """A helper class which contains the logic to translate the file into + a format BigQuery will accept.""" + + def parse_method(self, string_input): + """Translate CSV row to dictionary. + Args: + string_input: A comma separated list of values in the form of + name,surname + Example string_input: lorenzo,caggioni + Returns: + A dict mapping BigQuery column names as keys + example output: + { + 'name': 'mario', + 'surname': 'rossi', + 'age': 30 + } + """ + # Strip out carriage return, newline and quote characters. + values = re.split(",", re.sub('\r\n', '', re.sub('"', '', + string_input))) + row = dict( + zip(('name', 'surname', 'age'), + values)) + return row + + +class InjectTimestamp(beam.DoFn): + """A class which add a timestamp for each row. + Args: + element: A dictionary mapping BigQuery column names + Example: + { + 'name': 'mario', + 'surname': 'rossi', + 'age': 30 + } + Returns: + The input dictionary with a timestamp value added + Example: + { + 'name': 'mario', + 'surname': 'rossi', + 'age': 30 + '_TIMESTAMP': 1545730073 + } + """ + + def process(self, element): + import time + element['_TIMESTAMP'] = int(time.mktime(time.gmtime())) + return [element] + + +def run(argv=None): + """The main function which creates the pipeline and runs it.""" + + parser = argparse.ArgumentParser() + + parser.add_argument( + '--input', + dest='input', + required=False, + help='Input file to read. This can be a local file or ' + 'a file in a Google Storage Bucket.') + + parser.add_argument( + '--output', + dest='output', + required=False, + help='Output BQ table to write results to.') + + # Parse arguments from the command line. + known_args, pipeline_args = parser.parse_known_args(argv) + + # DataIngestion is a class we built in this script to hold the logic for + # transforming the file into a BigQuery table. + data_ingestion = DataIngestion() + + # Initiate the pipeline using the pipeline arguments + p = beam.Pipeline(options=PipelineOptions(pipeline_args)) + + (p + # Read the file. This is the source of the pipeline. + | 'Read from a File' >> beam.io.ReadFromText(known_args.input) + # Translates CSV row to a dictionary object consumable by BigQuery. + | 'String To BigQuery Row' >> + beam.Map(lambda s: data_ingestion.parse_method(s)) + # Add the timestamp on each row + | 'Inject Timestamp - ' >> beam.ParDo(InjectTimestamp()) + # Write data to Bigquery + | 'Write to BigQuery' >> beam.io.Write( + beam.io.BigQuerySink( + # BigQuery table name. + known_args.output, + # Bigquery table schema + schema='name:STRING,surname:STRING,age:NUMERIC,_TIMESTAMP:TIMESTAMP', + # Creates the table in BigQuery if it does not yet exist. + create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER, + # Deletes all data in the BigQuery table before writing. + write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE))) + p.run().wait_until_finish() + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + run() diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/diagram.png b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/diagram.png new file mode 100644 index 00000000..22e4518f Binary files /dev/null and b/data-solutions/gcs-to-bq-with-dataflow/scripts/data_ingestion/diagram.png differ diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/LICENSE b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/README.md b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/README.md new file mode 100644 index 00000000..43832cdf --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/README.md @@ -0,0 +1,17 @@ +# Create random Person PII data + +In this example you can find a Python script to generate Person PII data in a CSV file format. + +To know how to use the script run: + +```hcl +python3 person_details_generator.py --help +``` + +## Example +To create a file 'person.csv' with 10000 of random person details data you can run: +```hcl +python3 person_details_generator.py \ +--count 10000 \ +--output person.csv +``` \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/REQUIREMENTS.txt b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/REQUIREMENTS.txt new file mode 100644 index 00000000..b98f6609 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/REQUIREMENTS.txt @@ -0,0 +1 @@ +click \ No newline at end of file diff --git a/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/person_details_generator.py b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/person_details_generator.py new file mode 100644 index 00000000..f2694b30 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/scripts/person_details_generator/person_details_generator.py @@ -0,0 +1,47 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate random person PIIs based on arrays of names and surnames.""" + + +import click +import logging +import random + + +@click.command() +@click.option("--count", default=100, help="Number of generated names.") +@click.option("--output", default=False, help=( + "Name of the output file. Content will be overwritten. " + "If not defined, standard output will be used.")) +@click.option("--first_names", default="Lorenzo,Giacomo,Chiara,Miriam", help=( + "String of Names, comma separated. Default 'Lorenzo,Giacomo,Chiara,Miriam'")) +@click.option("--last_names", default="Rossi, Bianchi,Brambilla,Caggioni", help=( + "String of Names, comma separated. Default 'Rossi,Bianchi,Brambilla,Caggioni'")) +def main(count=100, output=False, first_names=None, last_names=None): + generated_names = "".join( + random.choice(first_names.split(',')) + "," + + random.choice(last_names.split(',')) + "," + + str(random.randint(1, 100)) + "\n" for _ in range(count))[:-1] + if output: + f = open(output, "w") + f.write(generated_names) + f.close() + else: + print(generated_names) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + main() diff --git a/data-solutions/gcs-to-bq-with-dataflow/variables.tf b/data-solutions/gcs-to-bq-with-dataflow/variables.tf new file mode 100644 index 00000000..a44f874a --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/variables.tf @@ -0,0 +1,76 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "billing_account" { + description = "Billing account id used as default for new projects." + type = string +} + +variable "location" { + description = "The location where resources will be deployed." + type = string + default = "europe" +} + +variable "project_service_name" { + description = "Name for the new Service Project." + type = string +} + +variable "project_kms_name" { + description = "Name for the new KMS Project." + type = string +} + +variable "region" { + description = "The region where resources will be deployed." + type = string + default = "europe-west1" +} + +variable "root_node" { + description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id." + type = string +} + +variable "vpc_name" { + description = "Name of the VPC created in the Service Project." + type = string + default = "local" +} + +variable "vpc_subnet_name" { + description = "Name of the subnet created in the Service Project." + type = string + default = "subnet" +} + +variable "vpc_ip_cidr_range" { + description = "Ip range used in the subnet deployef in the Service Project." + type = string + default = "10.0.0.0/20" +} + +variable "zone" { + description = "The zone where resources will be deployed." + type = string + default = "europe-west1-b" +} + +variable "ssh_source_ranges" { + description = "IP CIDR ranges that will be allowed to connect via SSH to the onprem instance." + type = list(string) + default = ["0.0.0.0/0"] +} diff --git a/data-solutions/gcs-to-bq-with-dataflow/versions.tf b/data-solutions/gcs-to-bq-with-dataflow/versions.tf new file mode 100644 index 00000000..057095c0 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-dataflow/versions.tf @@ -0,0 +1,17 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.12.6" +} diff --git a/foundations/business-units/main.tf b/foundations/business-units/main.tf index 1a599b8d..b30b5c46 100644 --- a/foundations/business-units/main.tf +++ b/foundations/business-units/main.tf @@ -29,14 +29,15 @@ module "shared-folder" { # Terraform project module "tf-project" { - source = "../../modules/project" - name = "terraform" - parent = module.shared-folder.id - prefix = var.prefix - billing_account = var.billing_account_id - iam_additive_members = { "roles/owner" = var.iam_terraform_owners } - iam_additive_roles = ["roles/owner"] - services = var.project_services + source = "../../modules/project" + name = "terraform" + parent = module.shared-folder.id + prefix = var.prefix + billing_account = var.billing_account_id + iam_additive_bindings = { + for name in var.iam_terraform_owners : (name) => ["roles/owner"] + } + services = var.project_services } # Bootstrap Terraform state GCS bucket @@ -115,6 +116,12 @@ module "audit-dataset" { project_id = module.audit-project.project_id id = "audit_export" friendly_name = "Audit logs export." + # disable delete on destroy for actual use + options = { + default_table_expiration_ms = null + default_partition_expiration_ms = null + delete_contents_on_destroy = true + } } module "audit-log-sinks" { @@ -140,12 +147,9 @@ module "shared-project" { parent = module.shared-folder.id prefix = var.prefix billing_account = var.billing_account_id - iam_additive_members = { - "roles/owner" = var.iam_shared_owners + iam_additive_bindings = { + for name in var.iam_shared_owners : (name) => ["roles/owner"] } - iam_additive_roles = [ - "roles/owner" - ] services = var.project_services } diff --git a/foundations/environments/README.md b/foundations/environments/README.md index 91ba7aa4..338c7286 100644 --- a/foundations/environments/README.md +++ b/foundations/environments/README.md @@ -39,12 +39,10 @@ If no shared services are needed, the shared service project module can of cours | root_node | Root node for the new hierarchy, either 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | | *audit_filter* | Audit log filter used for the log sink. | string | | ... | | *gcs_location* | GCS bucket location. | string | | EU | -| *iam_assets_editors* | Shared assets project editors, in IAM format. | list(string) | | [] | -| *iam_assets_owners* | Shared assets project owners, in IAM format. | list(string) | | [] | | *iam_audit_viewers* | Audit project viewers, in IAM format. | list(string) | | [] | | *iam_billing_config* | Control granting billing user role to service accounts. Target the billing account by default. | object({...}) | | ... | | *iam_folder_roles* | List of roles granted to each service account on its respective folder (excluding XPN roles). | list(string) | | ... | -| *iam_sharedsvc_owners* | Shared services project owners, in IAM format. | list(string) | | [] | +| *iam_shared_owners* | Shared services project owners, in IAM format. | list(string) | | [] | | *iam_terraform_owners* | Terraform project owners, in IAM format. | list(string) | | [] | | *iam_xpn_config* | Control granting Shared VPC creation roles to service accounts. Target the root node by default. | object({...}) | | ... | | *project_services* | Service APIs enabled by default in new projects. | list(string) | | ... | diff --git a/foundations/environments/main.tf b/foundations/environments/main.tf index ee55f161..90b4a8bf 100644 --- a/foundations/environments/main.tf +++ b/foundations/environments/main.tf @@ -19,14 +19,15 @@ # Terraform project module "tf-project" { - source = "../../modules/project" - name = "terraform" - parent = var.root_node - prefix = var.prefix - billing_account = var.billing_account_id - iam_additive_members = { "roles/owner" = var.iam_terraform_owners } - iam_additive_roles = ["roles/owner"] - services = var.project_services + source = "../../modules/project" + name = "terraform" + parent = var.root_node + prefix = var.prefix + billing_account = var.billing_account_id + iam_additive_bindings = { + for name in var.iam_terraform_owners : (name) => ["roles/owner"] + } + services = var.project_services } # per-environment service accounts @@ -130,6 +131,12 @@ module "audit-dataset" { project_id = module.audit-project.project_id id = "audit_export" friendly_name = "Audit logs export." + # disable delete on destroy for actual use + options = { + default_table_expiration_ms = null + default_partition_expiration_ms = null + delete_contents_on_destroy = true + } } module "audit-log-sinks" { @@ -156,12 +163,9 @@ module "sharedsvc-project" { parent = var.root_node prefix = var.prefix billing_account = var.billing_account_id - iam_additive_members = { - "roles/owner" = var.iam_sharedsvc_owners + iam_additive_bindings = { + for name in var.iam_shared_owners : (name) => ["roles/owner"] } - iam_additive_roles = [ - "roles/owner" - ] services = var.project_services } diff --git a/foundations/environments/variables.tf b/foundations/environments/variables.tf index a868140d..5cb74456 100644 --- a/foundations/environments/variables.tf +++ b/foundations/environments/variables.tf @@ -38,18 +38,6 @@ variable "gcs_location" { default = "EU" } -variable "iam_assets_editors" { - description = "Shared assets project editors, in IAM format." - type = list(string) - default = [] -} - -variable "iam_assets_owners" { - description = "Shared assets project owners, in IAM format." - type = list(string) - default = [] -} - variable "iam_audit_viewers" { description = "Audit project viewers, in IAM format." type = list(string) @@ -79,7 +67,7 @@ variable "iam_folder_roles" { ] } -variable "iam_sharedsvc_owners" { +variable "iam_shared_owners" { description = "Shared services project owners, in IAM format." type = list(string) default = [] diff --git a/infrastructure/hub-and-spoke-vpn/README.md b/infrastructure/hub-and-spoke-vpn/README.md index 2e39e68c..ab866ca2 100644 --- a/infrastructure/hub-and-spoke-vpn/README.md +++ b/infrastructure/hub-and-spoke-vpn/README.md @@ -42,7 +42,7 @@ If a single router and VPN gateway are used in the hub to manage all tunnels, pa | project_id | Project id for all resources. | string | ✓ | | | *bgp_asn* | BGP ASNs. | map(number) | | ... | | *bgp_custom_advertisements* | BGP custom advertisement IP CIDR ranges. | map(string) | | ... | -| *bgp_interface_ranges* | None | | | ... | +| *bgp_interface_ranges* | BGP interface IP CIDR ranges. | map(string) | | ... | | *ip_ranges* | IP CIDR ranges. | map(string) | | ... | | *regions* | VPC regions. | map(string) | | ... | diff --git a/infrastructure/onprem-google-access-dns/README.md b/infrastructure/onprem-google-access-dns/README.md index 2820c8f5..9f646104 100644 --- a/infrastructure/onprem-google-access-dns/README.md +++ b/infrastructure/onprem-google-access-dns/README.md @@ -153,16 +153,16 @@ The VPN used to connect to the on-premises environment does not account for HA, | project_id | Project id for all resources. | string | ✓ | | | *bgp_asn* | BGP ASNs. | map(number) | | ... | | *bgp_interface_ranges* | BGP interface IP CIDR ranges. | map(string) | | ... | +| *dns_forwarder_address* | Address of the DNS server used to forward queries from on-premises. | string | | 10.0.0.2 | +| *forwarder_address* | GCP DNS inbound policy forwarder address. | string | | 10.0.0.2 | | *ip_ranges* | IP CIDR ranges. | map(string) | | ... | | *region* | VPC region. | string | | europe-west1 | -| *resolver_address* | GCP DNS resolver address for the inbound policy. | string | | 10.0.0.2 | | *ssh_source_ranges* | IP CIDR ranges that will be allowed to connect via SSH to the onprem instance. | list(string) | | ["0.0.0.0/0"] | ## Outputs | name | description | sensitive | |---|---|:---:| -| foo | None | | | onprem-instance | Onprem instance details. | | | test-instance | Test instance details. | | diff --git a/infrastructure/onprem-google-access-dns/versions.tf b/infrastructure/onprem-google-access-dns/versions.tf index de5425c2..057095c0 100644 --- a/infrastructure/onprem-google-access-dns/versions.tf +++ b/infrastructure/onprem-google-access-dns/versions.tf @@ -13,5 +13,5 @@ # limitations under the License. terraform { - required_version = ">= 0.12" + required_version = ">= 0.12.6" } diff --git a/infrastructure/shared-vpc-gke/main.tf b/infrastructure/shared-vpc-gke/main.tf index 1c8d01bd..6d8ff1e2 100644 --- a/infrastructure/shared-vpc-gke/main.tf +++ b/infrastructure/shared-vpc-gke/main.tf @@ -17,6 +17,7 @@ ############################################################################### # the container.hostServiceAgentUser role is needed for GKE on shared VPC +# see: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#grant_host_service_agent_role module "project-host" { source = "../../modules/project" @@ -30,7 +31,7 @@ module "project-host" { ] iam_members = { "roles/container.hostServiceAgentUser" = [ - "serviceAccount:${module.project-svc-gke.gke_service_account}" + "serviceAccount:${module.project-svc-gke.service_accounts.robots.container-engine}" ] "roles/owner" = var.owners_host } @@ -81,12 +82,6 @@ module "project-svc-gke" { # Networking # ################################################################################ -# the service project GKE robot needs the `hostServiceAgent` role throughout -# the entire life of its clusters; the `iam_project_id` project output is used -# here to set the project id so that the VPC depends on that binding, and any -# cluster using it then also depends on it indirectly; you can of course use -# the `project_id` output instead if you don't care about destroying - # subnet IAM bindings control which identities can use the individual subnets module "vpc-shared" { @@ -122,16 +117,16 @@ module "vpc-shared" { iam_members = { "${var.region}/gce" = { "roles/compute.networkUser" = concat(var.owners_gce, [ - "serviceAccount:${module.project-svc-gce.cloudsvc_service_account}", + "serviceAccount:${module.project-svc-gce.service_accounts.cloud_services}", ]) } "${var.region}/gke" = { "roles/compute.networkUser" = concat(var.owners_gke, [ - "serviceAccount:${module.project-svc-gke.cloudsvc_service_account}", - "serviceAccount:${module.project-svc-gke.gke_service_account}", + "serviceAccount:${module.project-svc-gke.service_accounts.cloud_services}", + "serviceAccount:${module.project-svc-gke.service_accounts.robots.container-engine}", ]) "roles/compute.securityAdmin" = [ - "serviceAccount:${module.project-svc-gke.gke_service_account}", + "serviceAccount:${module.project-svc-gke.service_accounts.robots.container-engine}", ] } } diff --git a/modules/README.md b/modules/README.md index 418ed65e..4c73872a 100644 --- a/modules/README.md +++ b/modules/README.md @@ -21,13 +21,15 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google - [address reservation](./net-address) - [Cloud DNS](./dns) - [Cloud NAT](./net-cloudnat) +- [Cloud Endpoints](./endpoints) - [L4 Internal Load Balancer](./net-ilb) +- [Service Directory](./service-directory) - [VPC](./net-vpc) - [VPC firewall](./net-vpc-firewall) - [VPC peering](./net-vpc-peering) - [VPN static](./net-vpn-static) - [VPN dynamic](./net-vpn-dynamic) -- [VPN HA](./net-vpn-ha)) +- [VPN HA](./net-vpn-ha) - [ ] TODO: xLB modules ## Compute/Container @@ -41,9 +43,22 @@ Specific modules also offer support for non-authoritative bindings (e.g. `google ## Data - [BigQuery dataset](./bigquery-dataset) +- [Datafusion](./datafusion) - [GCS](./gcs) +- [Pub/Sub](./pubsub) +- [Bigtable instance](./bigtable-instance) + +## Development + +- [Artifact Registry](./artifact-registry) +- [Container Registry](./container-registry) +- [Source Repository](./source-repository) ## Security - [Cloud KMS](./kms) - [Secret Manager](./secret-manager) + +## Serverless + +- [Cloud Functions](./cloud-function) diff --git a/modules/__experimental/net-neg/README.md b/modules/__experimental/net-neg/README.md new file mode 100644 index 00000000..5a1e6a33 --- /dev/null +++ b/modules/__experimental/net-neg/README.md @@ -0,0 +1,46 @@ +# Network Endpoint Group Module + +This modules allows creating zonal network endpoint groups. + +Note: this module will integrated into a general-purpose load balancing module in the future. + +## Example +```hcl +module "neg" { + source = "./modules/net-neg" + project_id = "myproject" + name = "myneg" + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["europe-west1/default"] + zone = "europe-west1-b" + endpoints = [ + for instance in module.vm.instances : + { + instance = instance.name + port = 80 + ip_address = instance.network_interface[0].network_ip + } + ] +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| endpoints | List of (instance, port, address) of the NEG | list(object({...})) | ✓ | | +| name | NEG name | string | ✓ | | +| network | Name or self link of the VPC used for the NEG. Use the self link for Shared VPC. | string | ✓ | | +| project_id | NEG project id. | string | ✓ | | +| subnetwork | VPC subnetwork name or self link. | string | ✓ | | +| zone | NEG zone | string | ✓ | | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | Network endpoint group ID | | +| self_lnk | Network endpoint group self link | | +| size | Size of the network endpoint group | | + diff --git a/modules/__experimental/net-neg/main.tf b/modules/__experimental/net-neg/main.tf new file mode 100644 index 00000000..ceaa6d8f --- /dev/null +++ b/modules/__experimental/net-neg/main.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network_endpoint_group" "group" { + project = var.project_id + name = var.name + network = var.network + subnetwork = var.subnetwork + zone = var.zone +} + +resource "google_compute_network_endpoint" "endpoint" { + for_each = { for endpoint in var.endpoints : endpoint.instance => endpoint } + project = var.project_id + network_endpoint_group = google_compute_network_endpoint_group.group.name + instance = each.value.instance + port = each.value.port + ip_address = each.value.ip_address + zone = var.zone +} diff --git a/modules/__experimental/net-neg/outputs.tf b/modules/__experimental/net-neg/outputs.tf new file mode 100644 index 00000000..c579256b --- /dev/null +++ b/modules/__experimental/net-neg/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Network endpoint group ID" + value = google_compute_network_endpoint_group.group.name +} + +output "size" { + description = "Size of the network endpoint group" + value = google_compute_network_endpoint_group.group.size +} + +output "self_lnk" { + description = "Network endpoint group self link" + value = google_compute_network_endpoint_group.group.self_link +} diff --git a/modules/__experimental/net-neg/variables.tf b/modules/__experimental/net-neg/variables.tf new file mode 100644 index 00000000..fdc8c19f --- /dev/null +++ b/modules/__experimental/net-neg/variables.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "NEG project id." + type = string +} + +variable "name" { + description = "NEG name" + type = string +} + +variable "network" { + description = "Name or self link of the VPC used for the NEG. Use the self link for Shared VPC." + type = string +} + +variable "subnetwork" { + description = "VPC subnetwork name or self link." + type = string +} + +variable "zone" { + description = "NEG zone" + type = string +} + +variable "endpoints" { + description = "List of (instance, port, address) of the NEG" + type = list(object({ + instance = string + port = number + ip_address = string + })) +} diff --git a/modules/__experimental/net-neg/versions.tf b/modules/__experimental/net-neg/versions.tf new file mode 100644 index 00000000..bc4c2a9d --- /dev/null +++ b/modules/__experimental/net-neg/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/artifact-registry/README.md b/modules/artifact-registry/README.md new file mode 100644 index 00000000..480ea23a --- /dev/null +++ b/modules/artifact-registry/README.md @@ -0,0 +1,43 @@ +# Google Cloud Artifact Registry Module + +This module simplifies the creation of repositories using Google Cloud Artifact Registry. + +Note: Artifact Registry is still in beta, hence this module currently uses the beta provider. + +## Example + +```hcl +module "docker_artifact_registry" { + source = "./modules/artifact-registry" + project_id = "myproject" + location = "europe-west1" + format = "DOCKER" + id = "myregistry" + iam_roles = ["roles/artifactregistry.admin"] + iam_members = { + "roles/artifactregistry.admin" = ["group:cicd@example.com"] + } +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| id | Repository id | string | ✓ | | +| project_id | Registry project id. | string | ✓ | | +| *description* | An optional description for the repository | string | | Terraform-managed registry | +| *format* | Repository format. One of DOCKER or UNSPECIFIED | string | | DOCKER | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | | {} | +| *iam_roles* | List of roles used to set authoritative bindings. | list(string) | | [] | +| *labels* | Labels to be attached to the registry. | map(string) | | {} | +| *location* | Registry location. Use `gcloud beta artifacts locations list' to get valid values | string | | | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | Repository id | | +| name | Repository name | | + diff --git a/modules/artifact-registry/main.tf b/modules/artifact-registry/main.tf new file mode 100644 index 00000000..81c366f4 --- /dev/null +++ b/modules/artifact-registry/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_artifact_registry_repository" "registry" { + provider = google-beta + project = var.project_id + location = var.location + description = var.description + format = var.format + labels = var.labels + repository_id = var.id +} + +resource "google_artifact_registry_repository_iam_binding" "bindings" { + provider = google-beta + for_each = toset(var.iam_roles) + project = var.project_id + location = google_artifact_registry_repository.registry.location + repository = google_artifact_registry_repository.registry.name + role = each.value + members = lookup(var.iam_members, each.value, []) +} diff --git a/modules/artifact-registry/outputs.tf b/modules/artifact-registry/outputs.tf new file mode 100644 index 00000000..5800cb3f --- /dev/null +++ b/modules/artifact-registry/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Repository id" + value = google_artifact_registry_repository.registry.id +} + +output "name" { + description = "Repository name" + value = google_artifact_registry_repository.registry.name +} diff --git a/modules/artifact-registry/variables.tf b/modules/artifact-registry/variables.tf new file mode 100644 index 00000000..7aa8fdca --- /dev/null +++ b/modules/artifact-registry/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam_members" { + description = "Map of member lists used to set authoritative bindings, keyed by role." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "List of roles used to set authoritative bindings." + type = list(string) + default = [] +} + +variable "location" { + description = "Registry location. Use `gcloud beta artifacts locations list' to get valid values" + type = string + default = "" +} + +variable "project_id" { + description = "Registry project id." + type = string +} + +variable "labels" { + description = "Labels to be attached to the registry." + type = map(string) + default = {} +} + +variable "format" { + description = "Repository format. One of DOCKER or UNSPECIFIED" + type = string + default = "DOCKER" +} + +variable "description" { + description = "An optional description for the repository" + type = string + default = "Terraform-managed registry" +} + +variable "id" { + description = "Repository id" + type = string +} diff --git a/modules/artifact-registry/versions.tf b/modules/artifact-registry/versions.tf new file mode 100644 index 00000000..bc4c2a9d --- /dev/null +++ b/modules/artifact-registry/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index ecbbe8f1..20f946c0 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -20,7 +20,7 @@ The access variables are split into `access_roles` and `access_identities` varia ```hcl module "bigquery-dataset" { source = "./modules/bigquery-dataset" - project_id = "my-project + project_id = "my-project" id = "my-dataset" access_roles = { reader-group = { role = "READER", type = "group_by_email" } @@ -40,7 +40,7 @@ Dataset options are set via the `options` variable. all options must be specifie ```hcl module "bigquery-dataset" { source = "./modules/bigquery-dataset" - project_id = "my-project + project_id = "my-project" id = "my-dataset" options = { default_table_expiration_ms = 3600000 @@ -57,7 +57,7 @@ Tables are created via the `tables` variable, or the `view` variable for views. ```hcl module "bigquery-dataset" { source = "./modules/bigquery-dataset" - project_id = "my-project + project_id = "my-project" id = "my-dataset" tables = { table_a = { @@ -76,7 +76,7 @@ If partitioning is needed, populate the `partitioning` variable using either the ```hcl module "bigquery-dataset" { source = "./modules/bigquery-dataset" - project_id = "my-project + project_id = "my-project" id = "my-dataset" tables = { table_a = { @@ -99,7 +99,7 @@ To create views use the `view` variable. If you're querying a table created by t ```hcl module "bigquery-dataset" { source = "./modules/bigquery-dataset" - project_id = "my-project + project_id = "my-project" id = "my-dataset" tables = { table_a = { @@ -158,6 +158,3 @@ module "bigquery-dataset" { | views | View resources. | | -## TODO - -- [ ] add support for tables diff --git a/modules/bigtable-instance/README.md b/modules/bigtable-instance/README.md new file mode 100644 index 00000000..f63e1cf0 --- /dev/null +++ b/modules/bigtable-instance/README.md @@ -0,0 +1,65 @@ +# Google Cloud BigTable Module + +This module allows managing a single BigTable instance, including access configuration and tables. + +## TODO + +- [ ] support bigtable_gc_policy +- [ ] support bigtable_app_profile + +## Examples + +### Simple instance with access configuration + +```hcl + +module "big-table-instance" { + source = "./modules/bigtable-instance" + project_id = "my-project" + name = "instance" + cluster_id = "instance" + instance_type = "PRODUCTION" + tables = { + test1 = { table_options = null }, + test2 = { table_options = { + split_keys = ["a", "b", "c"] + column_family = null + } + } + } + iam_roles = ["viewer"] + iam_members = { + viewer = ["user:viewer@testdomain.com"] + } +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | The name of the Cloud Bigtable instance. | string | ✓ | | +| project_id | Id of the project where datasets will be created. | string | ✓ | | +| zone | The zone to create the Cloud Bigtable cluster in. | string | ✓ | | +| *cluster_id* | The ID of the Cloud Bigtable cluster. | string | | europe-west1 | +| *deletion_protection* | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | | | true | +| *display_name* | The human-readable display name of the Bigtable instance. | | | null | +| *iam_members* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved. | map(list(string)) | | {} | +| *iam_roles* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. | list(string) | | [] | +| *instance_type* | None | string | | DEVELOPMENT | +| *num_nodes* | The number of nodes in your Cloud Bigtable cluster. | number | | 1 | +| *storage_type* | The storage type to use. | string | | SSD | +| *table_options_defaults* | Default option of tables created in the BigTable instance. | object({...}) | | ... | +| *tables* | Tables to be created in the BigTable instance. | map(object({...})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | An identifier for the resource with format projects/{{project}}/instances/{{name}}. | | +| instance | BigTable intance. | | +| table_ids | Map of fully qualified table ids keyed by table name. | | +| tables | Table resources. | | + + diff --git a/modules/bigtable-instance/main.tf b/modules/bigtable-instance/main.tf new file mode 100644 index 00000000..0e7129ff --- /dev/null +++ b/modules/bigtable-instance/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + tables = { + for k, v in var.tables : k => v.table_options != null ? v.table_options : var.table_options_defaults + } + + iam_roles_bindings = { + for k in var.iam_roles : k => lookup(var.iam_members, k, []) + } +} + +resource "google_bigtable_instance" "default" { + project = var.project_id + name = var.name + cluster { + cluster_id = var.cluster_id + zone = var.zone + storage_type = var.storage_type + } + instance_type = var.instance_type + + display_name = var.display_name == null ? var.display_name : var.name + deletion_protection = var.deletion_protection +} + +resource "google_bigtable_instance_iam_binding" "default" { + for_each = local.iam_roles_bindings + + project = var.project_id + instance = google_bigtable_instance.default.name + role = "roles/bigtable.${each.key}" + members = each.value +} + +resource "google_bigtable_table" "default" { + for_each = local.tables + project = var.project_id + instance_name = google_bigtable_instance.default.name + name = each.key + split_keys = each.value.split_keys + + dynamic column_family { + for_each = each.value.column_family != null ? [""] : [] + + content { + family = each.value.column_family + } + } + + # lifecycle { + # prevent_destroy = true + # } +} diff --git a/modules/bigtable-instance/outputs.tf b/modules/bigtable-instance/outputs.tf new file mode 100644 index 00000000..2012b5c6 --- /dev/null +++ b/modules/bigtable-instance/outputs.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "An identifier for the resource with format projects/{{project}}/instances/{{name}}." + value = google_bigtable_instance.default.id + depends_on = [ + google_bigtable_instance_iam_binding, + google_bigtable_table + ] +} + +output "instance" { + description = "BigTable intance." + value = google_bigtable_instance.default + depends_on = [ + google_bigtable_instance_iam_binding, + google_bigtable_table + ] +} + +output "tables" { + description = "Table resources." + value = google_bigtable_table.default +} + +output "table_ids" { + description = "Map of fully qualified table ids keyed by table name." + value = { for k, v in google_bigtable_table.default : v.name => v.id } +} + + + diff --git a/modules/bigtable-instance/variables.tf b/modules/bigtable-instance/variables.tf new file mode 100644 index 00000000..982bfb8a --- /dev/null +++ b/modules/bigtable-instance/variables.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam_roles" { + description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members." + type = list(string) + default = [] +} + +variable "iam_members" { + description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved." + type = map(list(string)) + default = {} +} + +variable "cluster_id" { + description = "The ID of the Cloud Bigtable cluster." + type = string + default = "europe-west1" +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail." + default = true +} + +variable "display_name" { + description = "The human-readable display name of the Bigtable instance." + default = null +} + +variable "instance_type" { + description = "The instance type to create. One of \"DEVELOPMENT\" or \"PRODUCTION\". Defaults to \"DEVELOPMENT\"" + type = string + default = "DEVELOPMENT" +} + +variable "name" { + description = "The name of the Cloud Bigtable instance." + type = string +} + +variable "num_nodes" { + description = "The number of nodes in your Cloud Bigtable cluster." + type = number + default = 1 +} + +variable "project_id" { + description = "Id of the project where datasets will be created." + type = string +} + +variable "storage_type" { + description = "The storage type to use." + type = string + default = "SSD" +} + +variable "tables" { + description = "Tables to be created in the BigTable instance." + type = map(object({ + table_options = object({ + split_keys = list(string) + column_family = string + }) + })) + default = {} +} + +variable "table_options_defaults" { + description = "Default option of tables created in the BigTable instance." + type = object({ + split_keys = list(string) + column_family = string + }) + default = { + split_keys = [] + column_family = null + } +} + +variable "zone" { + description = "The zone to create the Cloud Bigtable cluster in." + type = string +} diff --git a/modules/bigtable-instance/versions.tf b/modules/bigtable-instance/versions.tf new file mode 100644 index 00000000..ce6918e0 --- /dev/null +++ b/modules/bigtable-instance/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/cloud-config-container/coredns/README.md b/modules/cloud-config-container/coredns/README.md index 5f8d5719..82ba51d8 100644 --- a/modules/cloud-config-container/coredns/README.md +++ b/modules/cloud-config-container/coredns/README.md @@ -24,7 +24,7 @@ This example will create a `cloud-config` that uses the module's defaults, creat ```hcl module "cos-coredns" { - source = "./modules/cos-container/coredns" + source = "./modules/cloud-config-container/coredns" } # use it as metadata in a compute instance or template @@ -40,8 +40,8 @@ This example will create a `cloud-config` using a custom CoreDNS configuration, ```hcl module "cos-coredns" { - source = "./modules/cos-container/coredns" - coredns_config = "./modules/cos-container/coredns/Corefile-hosts" + source = "./modules/cloud-config-container/coredns" + coredns_config = "./modules/cloud-config-container/coredns/Corefile-hosts" files = { "/etc/coredns/example.hosts" = { content = "127.0.0.2 foo.example.org foo" @@ -57,7 +57,7 @@ This example shows how to create the single instance optionally managed by the m ```hcl module "cos-coredns" { - source = "./modules/cos-container/coredns" + source = "./modules/cloud-config-container/coredns" test_instance = { project_id = "my-project" zone = "europe-west1-b" diff --git a/modules/cloud-config-container/cos-generic-metadata/README.md b/modules/cloud-config-container/cos-generic-metadata/README.md index 8e85e421..ad5ada41 100644 --- a/modules/cloud-config-container/cos-generic-metadata/README.md +++ b/modules/cloud-config-container/cos-generic-metadata/README.md @@ -21,9 +21,7 @@ module "cos-envoy" { container_args = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields" container_volumes = [ - { host = "/etc/envoy/envoy.yaml", - container = "/etc/envoy/envoy.yaml" - } + { host = "/etc/envoy/envoy.yaml", container = "/etc/envoy/envoy.yaml" } ] docker_args = "--network host --pid host" @@ -63,25 +61,26 @@ module "cos-envoy" { ## Variables -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| container\_image | Container image. | `string` | n/a | yes | -| boot\_commands | List of cloud-init `bootcmd`s | `list(string)` | `[]` | no | -| cloud\_config | Cloud config template path. If provided, takes precedence over all other arguments. | `string` | `null` | no | -| config\_variables | Additional variables used to render the template passed via `cloud_config` | `map(any)` | `{}` | no | -| container\_args | Arguments for container | `string` | `""` | no | -| container\_name | Name of the container to be run | `string` | `"container"` | no | -| container\_volumes | List of volumes | 
list(object({
    host      = string,
    container = string
  }))
| `[]` | no | -| docker\_args | Extra arguments to be passed for docker | `string` | `null` | no | -| file\_defaults | Default owner and permissions for files. | 
object({
    owner       = string
    permissions = string
  })
| 
{
  "owner": "root",
  "permissions": "0644"
}
| no | -| files | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | 
map(object({
    content     = string
    owner       = string
    permissions = string
  }))
| `{}` | no | -| gcp\_logging | Should container logs be sent to Google Cloud Logging | `bool` | `true` | no | -| run\_commands | List of cloud-init `runcmd`s | `list(string)` | `[]` | no | -| users | List of usernames to be created. If provided, first user will be used to run the container. | 
list(object({
    username = string,
    uid      = number,
  }))
| `[]` | no | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| container_image | Container image. | string | ✓ | | +| *authenticate_gcr* | Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined. | bool | | false | +| *boot_commands* | List of cloud-init `bootcmd`s | list(string) | | [] | +| *cloud_config* | Cloud config template path. If provided, takes precedence over all other arguments. | string | | null | +| *config_variables* | Additional variables used to render the template passed via `cloud_config` | map(any) | | {} | +| *container_args* | Arguments for container | string | | | +| *container_name* | Name of the container to be run | string | | container | +| *container_volumes* | List of volumes | list(object({...})) | | [] | +| *docker_args* | Extra arguments to be passed for docker | string | | null | +| *file_defaults* | Default owner and permissions for files. | object({...}) | | ... | +| *files* | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({...})) | | {} | +| *gcp_logging* | Should container logs be sent to Google Cloud Logging | bool | | true | +| *run_commands* | List of cloud-init `runcmd`s | list(string) | | [] | +| *users* | List of usernames to be created. If provided, first user will be used to run the container. | list(object({...})) | | ... | ## Outputs -| Name | Description | -|------|-------------| -| cloud\_config | Rendered cloud-config file to be passed as user-data instance metadata. | +| name | description | sensitive | +|---|---|:---:| +| cloud_config | Rendered cloud-config file to be passed as user-data instance metadata. | | diff --git a/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml b/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml index 9f8e38fa..2341415c 100644 --- a/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml +++ b/modules/cloud-config-container/cos-generic-metadata/cloud-config.yaml @@ -44,6 +44,10 @@ write_files: After=gcr-online.target docker.socket Wants=gcr-online.target docker.socket docker-events-collector.service [Service] +%{ if authenticate_gcr && length(users) > 0 ~} + Environment="HOME=/home/${users[0].username}" + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker +%{ endif ~} ExecStart=/usr/bin/docker run --rm --name=${container_name} \ %{ if length(users) > 0 ~} --user=${users[0].uid} \ diff --git a/modules/cloud-config-container/cos-generic-metadata/main.tf b/modules/cloud-config-container/cos-generic-metadata/main.tf index ee4c2ae0..85043b19 100644 --- a/modules/cloud-config-container/cos-generic-metadata/main.tf +++ b/modules/cloud-config-container/cos-generic-metadata/main.tf @@ -26,6 +26,7 @@ locals { gcp_logging = var.gcp_logging run_commands = var.run_commands users = var.users + authenticate_gcr = var.authenticate_gcr })) files = { for path, attrs in var.files : path => { diff --git a/modules/cloud-config-container/cos-generic-metadata/variables.tf b/modules/cloud-config-container/cos-generic-metadata/variables.tf index 1d54de00..4040fa44 100644 --- a/modules/cloud-config-container/cos-generic-metadata/variables.tf +++ b/modules/cloud-config-container/cos-generic-metadata/variables.tf @@ -108,3 +108,9 @@ variable "users" { default = [ ] } + +variable "authenticate_gcr" { + description = "Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined." + type = bool + default = false +} diff --git a/modules/cloud-config-container/envoy-traffic-director/README.md b/modules/cloud-config-container/envoy-traffic-director/README.md index 8cb39563..6e2dae0d 100644 --- a/modules/cloud-config-container/envoy-traffic-director/README.md +++ b/modules/cloud-config-container/envoy-traffic-director/README.md @@ -2,6 +2,10 @@ This module manages a `cloud-config` configuration that starts a containerized Envoy Proxy on Container Optimized OS connected to Traffic Director. The default configuration creates a reverse proxy exposed on the node's port 80. Traffic routing policies and management should be managed by other means via Traffic Director. +The generated cloud config is rendered in the `cloud_config` output, and is meant to be used in instances or instance templates via the `user-data` metadata. + +This module depends on the [`cos-generic-metadata` module](../cos-generic-metadata) being the parent folder. If you change its location be sure to adjust the `source` attribute in `main.tf`. + ## Examples ### Default configuration @@ -46,14 +50,14 @@ module "vm-cos" { ## Variables -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| envoy\_image | Envoy Proxy container image to use. | `string` | `"envoyproxy/envoy:v1.14.1"` | no | -| gcp\_logging | Should container logs be sent to Google Cloud Logging | `bool` | `true` | no | +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| *envoy_image* | Envoy Proxy container image to use. | string | | envoyproxy/envoy:v1.14.1 | +| *gcp_logging* | Should container logs be sent to Google Cloud Logging | bool | | true | ## Outputs -| Name | Description | -|------|-------------| -| cloud\_config | Rendered cloud-config file to be passed as user-data instance metadata. | +| name | description | sensitive | +|---|---|:---:| +| cloud_config | Rendered cloud-config file to be passed as user-data instance metadata. | | diff --git a/modules/cloud-config-container/envoy-traffic-director/main.tf b/modules/cloud-config-container/envoy-traffic-director/main.tf index 768ed5af..0b390d8c 100644 --- a/modules/cloud-config-container/envoy-traffic-director/main.tf +++ b/modules/cloud-config-container/envoy-traffic-director/main.tf @@ -15,7 +15,7 @@ */ module "cos-envoy-td" { - source = "./modules/cos-generic-metadata" + source = "../cos-generic-metadata" boot_commands = [ "systemctl start node-problem-detector", @@ -26,9 +26,7 @@ module "cos-envoy-td" { container_args = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields" container_volumes = [ - { host = "/etc/envoy/envoy.yaml", - container = "/etc/envoy/envoy.yaml" - } + { host = "/etc/envoy/envoy.yaml", container = "/etc/envoy/envoy.yaml" } ] docker_args = "--network host --pid host" diff --git a/modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata b/modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata deleted file mode 120000 index 66c564ef..00000000 --- a/modules/cloud-config-container/envoy-traffic-director/modules/cos-generic-metadata +++ /dev/null @@ -1 +0,0 @@ -../../cos-generic-metadata \ No newline at end of file diff --git a/modules/cloud-config-container/nginx/cloud-config.yaml b/modules/cloud-config-container/nginx/cloud-config.yaml index 83ceab81..18d8051e 100644 --- a/modules/cloud-config-container/nginx/cloud-config.yaml +++ b/modules/cloud-config-container/nginx/cloud-config.yaml @@ -74,5 +74,4 @@ bootcmd: runcmd: - iptables -I INPUT 1 -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT - systemctl daemon-reload - - systemctl restart systemd-resolved.service - - systemctl start nginx \ No newline at end of file + - systemctl start nginx diff --git a/modules/cloud-config-container/onprem/README.md b/modules/cloud-config-container/onprem/README.md index 260aad1b..a4a6c9ab 100644 --- a/modules/cloud-config-container/onprem/README.md +++ b/modules/cloud-config-container/onprem/README.md @@ -4,11 +4,11 @@ This module manages a `cloud-config` configuration that starts an emulated on-pr The emulated on-premises infrastructure is composed of: -- a Strongswan container managing the VPN tunnel to GCP +- a [Strongswan container](./docker-images/strongswan) managing the VPN tunnel to GCP - an optional Bird container managing the BGP session - a CoreDNS container servng local DNS and forwarding to GCP - an Nginx container serving a simple static web page -- a generic Linux container used as a jump host inside the on-premises network +- a [generic Linux container](./docker-images/toolbox) used as a jump host inside the on-premises network A [complete scenario using this module](../../../infrastructure/onprem-google-access-dns) is available in the infrastructure examples. diff --git a/modules/cloud-config-container/onprem/docker-images/README.md b/modules/cloud-config-container/onprem/docker-images/README.md new file mode 100644 index 00000000..e9342f7e --- /dev/null +++ b/modules/cloud-config-container/onprem/docker-images/README.md @@ -0,0 +1,3 @@ +# Supporting container images + +The images in this folder are used by the [`onprem` module](../). \ No newline at end of file diff --git a/docker-images/strongswan/Dockerfile b/modules/cloud-config-container/onprem/docker-images/strongswan/Dockerfile similarity index 100% rename from docker-images/strongswan/Dockerfile rename to modules/cloud-config-container/onprem/docker-images/strongswan/Dockerfile diff --git a/docker-images/strongswan/README.md b/modules/cloud-config-container/onprem/docker-images/strongswan/README.md similarity index 88% rename from docker-images/strongswan/README.md rename to modules/cloud-config-container/onprem/docker-images/strongswan/README.md index 42225416..cc6eca12 100644 --- a/docker-images/strongswan/README.md +++ b/modules/cloud-config-container/onprem/docker-images/strongswan/README.md @@ -1,9 +1,14 @@ # StrongSwan docker container -### [strongSwan](https://www.strongswan.org/) is an OpenSource IPsec-based VPN Solution +## Build + +```bash +gcloud builds submit . --config=cloudbuild.yaml +``` + +## Docker compose example -### Docker compose example ```yaml version: "3" services: @@ -37,8 +42,3 @@ services: - "/var/lib/docker-compose/onprem/bird/bird.conf:/etc/bird/bird.conf:ro" ``` - -### Build -```bash -gcloud builds submit . --config=cloudbuild.yaml -``` diff --git a/docker-images/strongswan/cloudbuild.yaml b/modules/cloud-config-container/onprem/docker-images/strongswan/cloudbuild.yaml similarity index 100% rename from docker-images/strongswan/cloudbuild.yaml rename to modules/cloud-config-container/onprem/docker-images/strongswan/cloudbuild.yaml diff --git a/docker-images/strongswan/entrypoint.sh b/modules/cloud-config-container/onprem/docker-images/strongswan/entrypoint.sh similarity index 100% rename from docker-images/strongswan/entrypoint.sh rename to modules/cloud-config-container/onprem/docker-images/strongswan/entrypoint.sh diff --git a/docker-images/strongswan/ipsec-vti.sh b/modules/cloud-config-container/onprem/docker-images/strongswan/ipsec-vti.sh similarity index 100% rename from docker-images/strongswan/ipsec-vti.sh rename to modules/cloud-config-container/onprem/docker-images/strongswan/ipsec-vti.sh diff --git a/docker-images/toolbox/Dockerfile b/modules/cloud-config-container/onprem/docker-images/toolbox/Dockerfile similarity index 100% rename from docker-images/toolbox/Dockerfile rename to modules/cloud-config-container/onprem/docker-images/toolbox/Dockerfile diff --git a/docker-images/toolbox/README.md b/modules/cloud-config-container/onprem/docker-images/toolbox/README.md similarity index 97% rename from docker-images/toolbox/README.md rename to modules/cloud-config-container/onprem/docker-images/toolbox/README.md index 6e2f70c2..6daada8f 100644 --- a/docker-images/toolbox/README.md +++ b/modules/cloud-config-container/onprem/docker-images/toolbox/README.md @@ -3,7 +3,7 @@ Lightweight container with some basic console tools used for testing and probing. -## Building +## Build ```bash gcloud builds submit . --config=cloudbuild.yaml diff --git a/docker-images/toolbox/cloudbuild.yaml b/modules/cloud-config-container/onprem/docker-images/toolbox/cloudbuild.yaml similarity index 100% rename from docker-images/toolbox/cloudbuild.yaml rename to modules/cloud-config-container/onprem/docker-images/toolbox/cloudbuild.yaml diff --git a/docker-images/toolbox/entrypoint.sh b/modules/cloud-config-container/onprem/docker-images/toolbox/entrypoint.sh similarity index 100% rename from docker-images/toolbox/entrypoint.sh rename to modules/cloud-config-container/onprem/docker-images/toolbox/entrypoint.sh diff --git a/modules/cloud-function/README.md b/modules/cloud-function/README.md new file mode 100644 index 00000000..86ae486e --- /dev/null +++ b/modules/cloud-function/README.md @@ -0,0 +1,162 @@ +# Cloud Function Module + +Cloud Function management, with support for IAM roles and optional bucket creation. + +The GCS object used for deployment uses a hash of the bundle zip contents in its name, which ensures change tracking and avoids recreating the function if the GCS object is deleted and needs recreating. + +## TODO + +- [ ] add support for `ingress_settings` +- [ ] add support for `vpc_connector` and `vpc_connector_egress_settings` +- [ ] add support for `source_repository` + +## Examples + +### HTTP trigger + +This deploys a Cloud Function with an HTTP endpoint, using a pre-existing GCS bucket for deployment, setting the service account to the Cloud Function default one, and delegating access control to the containing project. + +```hcl +module "cf-http" { + source = "../modules/net-cloudnat" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "my-cf-source-folder + output_path = "bundle.zip" + } +} +``` + +### Non-HTTP triggers + +Other trigger types other than HTTP are configured via the `trigger_config` variable. This example shows a PubSub trigger. + +```hcl +module "cf-http" { + source = "../modules/net-cloudnat" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "my-cf-source-folder + output_path = "bundle.zip" + } + trigger_config = { + event = "google.pubsub.topic.publish" + resource = local.my-topic + retry = null + } +} +``` + +### Controlling HTTP access + +To allow anonymous access to the function, grant the `roles/cloudfunctions.invoker` role to the special `allUsers` identifier. Use specific identities (service accounts, groups, etc.) instead of `allUsers` to only allow selective access. + +```hcl +module "cf-http" { + source = "../modules/net-cloudnat" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "my-cf-source-folder + output_path = "bundle.zip" + } + iam_roles = ["roles/cloudfunctions.invoker"] + iam_members = { + "roles/cloudfunctions.invoker" = ["allUsers"] + } +} +``` + +### GCS bucket creation + +You can have the module auto-create the GCS bucket used for deployment via the `bucket_config` variable. Setting `bucket_config.location` to `null` will also use the function region for GCS. + +```hcl +module "cf-http" { + source = "../modules/net-cloudnat" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bucket_config = { + location = null + lifecycle_delete_age = 1 + } + bundle_config = { + source_dir = "my-cf-source-folder + output_path = "bundle.zip" + } +} +``` + +### Service account management + +To use a custom service account managed by the module, set `service_account_create` to `true` and leave `service_account` set to `null` value (default). + +```hcl +module "cf-http" { + source = "../modules/net-cloudnat" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "my-cf-source-folder + output_path = "bundle.zip" + } + service_account_create = true +} +``` + +To use an externally managed service account, pass its email in `service_account` and leave `service_account_create` to `false` (the default). + +```hcl +module "cf-http" { + source = "../modules/net-cloudnat" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "my-cf-source-folder + output_path = "bundle.zip" + } + service_account = local.service_account_email +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| bucket_name | Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null. | string | ✓ | | +| bundle_config | Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null. | object({...}) | ✓ | | +| name | Name used for cloud function and associated resources. | string | ✓ | | +| project_id | Project id used for all resources. | string | ✓ | | +| *bucket_config* | Enable and configure auto-created bucket. Set fields to null to use defaults. | object({...}) | | null | +| *environment_variables* | Cloud function environment variables. | map(string) | | {} | +| *function_config* | Cloud function configuration. | object({...}) | | ... | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. Ignored for template use. | map(list(string)) | | {} | +| *iam_roles* | List of roles used to set authoritative bindings. Ignored for template use. | list(string) | | [] | +| *labels* | Resource labels | map(string) | | {} | +| *prefix* | Optional prefix used for resource names. | string | | null | +| *region* | Region used for all resources. | string | | europe-west1 | +| *service_account* | Service account email. Unused if service account is auto-created. | string | | null | +| *service_account_create* | Auto-create service account. | bool | | false | +| *trigger_config* | Function trigger configuration. Leave null for HTTP trigger. | object({...}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| bucket | Bucket resource (only if auto-created). | | +| bucket_name | Bucket name. | | +| function | Cloud function resources. | | +| function_name | Cloud function name. | | +| service_account | Service account resource. | | +| service_account_email | Service account email. | | +| service_account_iam_email | Service account email. | | + diff --git a/modules/cloud-function/main.tf b/modules/cloud-function/main.tf new file mode 100644 index 00000000..a668a8bc --- /dev/null +++ b/modules/cloud-function/main.tf @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + bucket = ( + var.bucket_name != null + ? var.bucket_name + : ( + length(google_storage_bucket.bucket) > 0 + ? google_storage_bucket.bucket[0].name + : null + ) + ) + prefix = var.prefix == null ? "" : "${var.prefix}-" + service_account_email = ( + var.service_account_create + ? ( + length(google_service_account.service_account) > 0 + ? google_service_account.service_account[0].email + : null + ) + : var.service_account + ) +} + +resource "google_cloudfunctions_function" "function" { + project = var.project_id + region = var.region + name = "${local.prefix}${var.name}" + description = "Terraform managed." + runtime = var.function_config.runtime + available_memory_mb = var.function_config.memory + max_instances = var.function_config.instances + timeout = var.function_config.timeout + entry_point = var.function_config.entry_point + environment_variables = var.environment_variables + service_account_email = local.service_account_email + source_archive_bucket = local.bucket + source_archive_object = google_storage_bucket_object.bundle.name + labels = var.labels + trigger_http = var.trigger_config == null ? true : null + + dynamic event_trigger { + for_each = var.trigger_config == null ? [] : [""] + content { + event_type = var.trigger_config.event + resource = var.trigger_config.resource + dynamic failure_policy { + for_each = var.trigger_config.retry == null ? [] : [""] + content { + retry = var.trigger_config.retry + } + } + } + } + +} + +resource "google_cloudfunctions_function_iam_binding" "default" { + for_each = toset(var.iam_roles) + project = var.project_id + region = var.region + cloud_function = google_cloudfunctions_function.function.name + role = each.value + members = try(var.iam_members[each.value], {}) +} + +resource "google_storage_bucket" "bucket" { + count = var.bucket_config == null ? 0 : 1 + project = var.project_id + name = "${local.prefix}${var.bucket_name}" + location = ( + var.bucket_config.location == null + ? var.region + : var.bucket_config.location + ) + labels = var.labels + + dynamic lifecycle_rule { + for_each = var.bucket_config.lifecycle_delete_age == null ? [] : [""] + content { + action { type = "Delete" } + condition { age = var.bucket_config.lifecycle_delete_age } + } + } +} + +resource "google_storage_bucket_object" "bundle" { + name = "bundle-${data.archive_file.bundle.output_md5}.zip" + bucket = local.bucket + source = data.archive_file.bundle.output_path +} + +data "archive_file" "bundle" { + type = "zip" + source_dir = var.bundle_config.source_dir + output_path = ( + var.bundle_config.output_path == null + ? "/tmp/bundle.zip" + : var.bundle_config.output_path + ) +} + +resource "google_service_account" "service_account" { + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "tf-cf-${var.name}" + display_name = "Terraform Cloud Function ${var.name}." +} diff --git a/modules/cloud-function/outputs.tf b/modules/cloud-function/outputs.tf new file mode 100644 index 00000000..43e0eda7 --- /dev/null +++ b/modules/cloud-function/outputs.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket" { + description = "Bucket resource (only if auto-created)." + value = var.bucket_config == null ? null : google_storage_bucket.bucket.0 +} + +output "bucket_name" { + description = "Bucket name." + value = local.bucket +} + +output "function" { + description = "Cloud function resources." + value = google_cloudfunctions_function.function +} + +output "function_name" { + description = "Cloud function name." + value = google_cloudfunctions_function.function.name +} + +output "service_account" { + description = "Service account resource." + value = ( + var.service_account_create ? google_service_account.service_account[0] : null + ) +} + +output "service_account_email" { + description = "Service account email." + value = local.service_account_email +} + +output "service_account_iam_email" { + description = "Service account email." + value = join("", [ + "serviceAccount:", + local.service_account_email == null ? "" : local.service_account_email + ]) +} diff --git a/modules/cloud-function/variables.tf b/modules/cloud-function/variables.tf new file mode 100644 index 00000000..83c8c048 --- /dev/null +++ b/modules/cloud-function/variables.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_config" { + description = "Enable and configure auto-created bucket. Set fields to null to use defaults." + type = object({ + location = string + lifecycle_delete_age = number + }) + default = null +} + +variable "bucket_name" { + description = "Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null." + type = string +} + +variable "bundle_config" { + description = "Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null." + type = object({ + source_dir = string + output_path = string + }) +} + +variable "environment_variables" { + description = "Cloud function environment variables." + type = map(string) + default = {} +} + +variable "iam_members" { + description = "Map of member lists used to set authoritative bindings, keyed by role. Ignored for template use." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "List of roles used to set authoritative bindings. Ignored for template use." + type = list(string) + default = [] +} + +variable "function_config" { + description = "Cloud function configuration." + type = object({ + entry_point = string + instances = number + memory = number + runtime = string + timeout = number + }) + default = { + entry_point = "main" + instances = 1 + memory = 256 + runtime = "python37" + timeout = 180 + } +} + +variable "labels" { + description = "Resource labels" + type = map(string) + default = {} +} + +variable "name" { + description = "Name used for cloud function and associated resources." + type = string +} + +variable "prefix" { + description = "Optional prefix used for resource names." + type = string + default = null +} + +variable "project_id" { + description = "Project id used for all resources." + type = string +} + +variable "region" { + description = "Region used for all resources." + type = string + default = "europe-west1" +} + +variable "service_account" { + description = "Service account email. Unused if service account is auto-created." + type = string + default = null +} + +variable "service_account_create" { + description = "Auto-create service account." + type = bool + default = false +} + +variable "trigger_config" { + description = "Function trigger configuration. Leave null for HTTP trigger." + type = object({ + event = string + resource = string + retry = bool + }) + default = null +} diff --git a/modules/cloud-function/versions.tf b/modules/cloud-function/versions.tf new file mode 100644 index 00000000..bc4c2a9d --- /dev/null +++ b/modules/cloud-function/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/compute-mig/README.md b/modules/compute-mig/README.md index 918a2a3d..beada11f 100644 --- a/modules/compute-mig/README.md +++ b/modules/compute-mig/README.md @@ -165,7 +165,7 @@ module "nginx-mig" { | project_id | Project id. | string | ✓ | | | *auto_healing_policies* | Auto-healing policies for this group. | object({...}) | | null | | *autoscaler_config* | Optional autoscaler configuration. Only one of 'cpu_utilization_target' 'load_balancing_utilization_target' or 'metric' can be not null. | object({...}) | | null | -| *health_check_config* | Optional auto-created helth check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | object({...}) | | null | +| *health_check_config* | Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | object({...}) | | null | | *named_ports* | Named ports. | map(number) | | null | | *regional* | Use regional instance group. When set, `location` should be set to the region. | bool | | false | | *target_pools* | Optional list of URLs for target pools to which new instances in the group are added. | list(string) | | [] | diff --git a/modules/compute-vm/README.md b/modules/compute-vm/README.md index 14fb10dc..91b35abb 100644 --- a/modules/compute-vm/README.md +++ b/modules/compute-vm/README.md @@ -31,12 +31,57 @@ module "simple-vm-example" { } ``` +### Disk encryption with Cloud KMS + +This example shows how to control disk encryption via the the `encryption` variable, in this case the self link to a KMS CryptoKey that will be used to encrypt boot and attached disk. Managing the key with the `../kms` module is of course possible, but is not shown here. + +```hcl +module "kms-vm-example" { + source = "../modules/compute-vm" + project_id = local.project_id + region = local.region + zone = local.zone + name = "kms-test" + network_interfaces = [{ + network = local.network_self_link, + subnetwork = local.subnet_self_link, + nat = false, + addresses = null + }] + attached_disks = [ + { + name = "attached-disk" + size = 10 + image = null + options = { + auto_delete = true + mode = null + source = null + type = null + } + } + ] + service_account_create = true + instance_count = 1 + boot_disk = { + image = "projects/debian-cloud/global/images/family/debian-10" + type = "pd-ssd" + size = 10 + } + encryption = { + encrypt_boot = true + disk_encryption_key_raw = null + kms_key_self_link = local.kms_key.self_link + } +} +``` + ### Instance template This example shows how to use the module to manage an instance template that defines an additional attached disk for each instance, and overrides defaults for the boot disk image and service account. ```hcl -module "debian-test" { +module "cos-test" { source = "../modules/compute-vm" project_id = "my-project" region = "europe-west1" @@ -86,11 +131,10 @@ module "instance-group" { } service_account = local.service_account_email service_account_scopes = ["https://www.googleapis.com/auth/cloud-platform"] - use_instance_template = true metadata = { user-data = local.cloud_config } - group = {} + group = { named_ports = {} } } ``` @@ -108,8 +152,11 @@ module "instance-group" { | *attached_disk_defaults* | Defaults for attached disks options. | object({...}) | | ... | | *attached_disks* | Additional disks, if options is null defaults will be used in its place. | list(object({...})) | | [] | | *boot_disk* | Boot disk properties. | object({...}) | | ... | +| *encryption* | Encryption options. Only one of kms_key_self_link and disk_encryption_key_raw may be set. If needed, you can specify to encrypt or not the boot disk. | object({...}) | | null | | *group* | Define this variable to create an instance group for instances. Disabled for template use. | object({...}) | | null | | *hostname* | Instance FQDN name. | string | | null | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. Ignored for template use. | map(list(string)) | | {} | +| *iam_roles* | List of roles used to set authoritative bindings. Ignored for template use. | list(string) | | [] | | *instance_count* | Number of instances to create (only for non-template usage). | number | | 1 | | *instance_type* | Instance type. | string | | f1-micro | | *labels* | Instance labels. | map(string) | | {} | diff --git a/modules/compute-vm/main.tf b/modules/compute-vm/main.tf index e97d576c..41804461 100644 --- a/modules/compute-vm/main.tf +++ b/modules/compute-vm/main.tf @@ -25,6 +25,10 @@ locals { for pair in setproduct(keys(local.names), keys(local.attached_disks)) : "${pair[0]}-${pair[1]}" => { name = pair[0], disk_name = pair[1] } } + iam_roles = var.use_instance_template ? {} : { + for pair in setproduct(var.iam_roles, keys(local.names)) : + "${pair.0}/${pair.1}" => { role = pair.0, name = pair.1 } + } names = ( var.use_instance_template ? { "${var.name}" = 0 } @@ -66,6 +70,14 @@ resource "google_compute_disk" "disks" { disk_type = local.attached_disks[each.value.disk_name].options.type image = local.attached_disks[each.value.disk_name].image }) + dynamic disk_encryption_key { + for_each = var.encryption != null ? [""] : [] + + content { + raw_key = var.encryption.disk_encryption_key_raw + kms_key_self_link = var.encryption.kms_key_self_link + } + } } resource "google_compute_instance" "default" { @@ -103,6 +115,8 @@ resource "google_compute_instance" "default" { image = var.boot_disk.image size = var.boot_disk.size } + disk_encryption_key_raw = var.encryption != null ? var.encryption.disk_encryption_key_raw : null + kms_key_self_link = var.encryption != null ? var.encryption.kms_key_self_link : null } dynamic network_interface { @@ -121,7 +135,7 @@ resource "google_compute_instance" "default" { iterator = nat_addresses content { nat_ip = nat_addresses.value == null ? null : ( - length(nat_addresses.value) == 0 ? null : nat_addresses.value[each.value] + length(nat_addresses.value) == 0 ? null : nat_addresses.value.external[each.value] ) } } @@ -154,6 +168,16 @@ resource "google_compute_instance" "default" { } +resource "google_compute_instance_iam_binding" "default" { + for_each = local.iam_roles + project = var.project_id + zone = var.zone + instance_name = each.value.name + role = each.value.role + members = lookup(var.iam_members, each.value.role, []) + depends_on = [google_compute_instance.default] +} + resource "google_compute_instance_template" "default" { count = var.use_instance_template ? 1 : 0 project = var.project_id diff --git a/modules/compute-vm/variables.tf b/modules/compute-vm/variables.tf index 3593fef2..2102d2fb 100644 --- a/modules/compute-vm/variables.tf +++ b/modules/compute-vm/variables.tf @@ -60,6 +60,16 @@ variable "boot_disk" { } } +variable "encryption" { + description = "Encryption options. Only one of kms_key_self_link and disk_encryption_key_raw may be set. If needed, you can specify to encrypt or not the boot disk." + type = object({ + encrypt_boot = bool + disk_encryption_key_raw = string + kms_key_self_link = string + }) + default = null +} + variable "group" { description = "Define this variable to create an instance group for instances. Disabled for template use." type = object({ @@ -74,6 +84,18 @@ variable "hostname" { default = null } +variable "iam_members" { + description = "Map of member lists used to set authoritative bindings, keyed by role. Ignored for template use." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "List of roles used to set authoritative bindings. Ignored for template use." + type = list(string) + default = [] +} + variable "instance_count" { description = "Number of instances to create (only for non-template usage)." type = number diff --git a/modules/container-registry/README.md b/modules/container-registry/README.md new file mode 100644 index 00000000..167f79b3 --- /dev/null +++ b/modules/container-registry/README.md @@ -0,0 +1,34 @@ +# Google Cloud Container Registry Module + +This module simplifies the creation of GCS buckets used by Google Container Registry. + +## Example + +```hcl +module "container_registry" { + source = "../../modules/container-registry" + project_id = "myproject" + location = "EU" + iam_roles = ["roles/storage.admin"] + iam_members = { + "roles/storage.admin" = ["group:cicd@example.com"] + } +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| project_id | Registry project id. | string | ✓ | | +| *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | | null | +| *iam_roles* | List of roles used to set authoritative bindings. | list(string) | | null | +| *location* | Registry location. Can be US, EU, ASIA or empty | string | | | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| bucket_id | ID of the GCS bucket created | | + diff --git a/modules/container-registry/main.tf b/modules/container-registry/main.tf new file mode 100644 index 00000000..073e2995 --- /dev/null +++ b/modules/container-registry/main.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_registry" "registry" { + project = var.project_id + location = var.location +} + +resource "google_storage_bucket_iam_binding" "bindings" { + for_each = toset(var.iam_roles) + bucket = google_container_registry.registry.id + role = each.value + members = lookup(var.iam_members, each.value, []) +} diff --git a/modules/container-registry/outputs.tf b/modules/container-registry/outputs.tf new file mode 100644 index 00000000..cbd09710 --- /dev/null +++ b/modules/container-registry/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket_id" { + description = "ID of the GCS bucket created" + value = google_container_registry.registry.id +} diff --git a/modules/container-registry/variables.tf b/modules/container-registry/variables.tf new file mode 100644 index 00000000..15074aca --- /dev/null +++ b/modules/container-registry/variables.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam_members" { + description = "Map of member lists used to set authoritative bindings, keyed by role." + type = map(list(string)) + default = null +} + +variable "iam_roles" { + description = "List of roles used to set authoritative bindings." + type = list(string) + default = null +} + +variable "location" { + description = "Registry location. Can be US, EU, ASIA or empty" + type = string + default = "" +} + +variable "project_id" { + description = "Registry project id." + type = string +} diff --git a/modules/container-registry/versions.tf b/modules/container-registry/versions.tf new file mode 100644 index 00000000..bc4c2a9d --- /dev/null +++ b/modules/container-registry/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/datafusion/README.md b/modules/datafusion/README.md new file mode 100644 index 00000000..75fddc65 --- /dev/null +++ b/modules/datafusion/README.md @@ -0,0 +1,63 @@ +# Google Cloud Data Fusion Module + +This module allows simple management of ['Google Data Fusion'](https://cloud.google.com/data-fusion) instances. It supports creating Basic or Enterprise, public or private instances. + +## Examples + +## Auto-managed IP allocation + +```hcl +module "datafusion" { + source = "./modules/datafusion" + name = "my-datafusion" + region = "europe-west1" + project_id = "my-project" + network = "my-network-name" +} +``` + +### Externally managed IP allocation + +```hcl +module "datafusion" { + source = "./modules/datafusion" + name = "my-datafusion" + region = "europe-west1" + project_id = "my-project" + network = "my-network-name" + ip_allocation_create = false + ip_allocation = "10.0.0.0/22" +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Name of the DataFusion instance. | string | ✓ | | +| network | Name of the network in the project with which the tenant project will be peered for executing pipelines in the form of projects/{project-id}/global/networks/{network} | string | ✓ | | +| project_id | Project ID. | string | ✓ | | +| region | DataFusion region. | string | ✓ | | +| *description* | DataFuzion instance description. | string | | Terraform managed. | +| *enable_stackdriver_logging* | Option to enable Stackdriver Logging. | bool | | false | +| *enable_stackdriver_monitoring* | Option to enable Stackdriver Monitorig. | bool | | false | +| *firewall_create* | Create Network firewall rules to enable SSH. | bool | | true | +| *ip_allocation* | Ip allocated for datafusion instance when not using the auto created one and created outside of the module. | string | | null | +| *ip_allocation_create* | Create Ip range for datafusion instance. | bool | | true | +| *labels* | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| *network_peering* | Create Network peering between project and DataFusion tenant project. | bool | | true | +| *private_instance* | Create private instance. | bool | | true | +| *type* | Datafusion Instance type. It can be BASIC or ENTERPRISE (default value). | string | | ENTERPRISE | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | DataFusion instance ID. | | +| ip_allocation | IP range reserved for Data Fusion instance in case of a private instance. | | +| resource | DataFusion resource. | | +| service_account | DataFusion Service Account. | | +| service_endpoint | DataFusion Service Endpoint. | | +| version | DataFusion version. | | + diff --git a/modules/datafusion/main.tf b/modules/datafusion/main.tf new file mode 100644 index 00000000..9161c013 --- /dev/null +++ b/modules/datafusion/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix_length = 22 + ip_allocation = ( + var.ip_allocation_create + ? "${google_compute_global_address.default[0].address}/${local.prefix_length}" + : var.ip_allocation + ) + tenant_project = regex( + "cloud-datafusion-management-sa@([\\w-]+).iam.gserviceaccount.com", + google_data_fusion_instance.default.service_account + )[0] +} + +resource "google_compute_global_address" "default" { + count = var.ip_allocation_create ? 1 : 0 + project = var.project_id + name = "cdf-${var.name}" + address_type = "INTERNAL" + purpose = "VPC_PEERING" + prefix_length = local.prefix_length + network = var.network +} + +resource "google_compute_network_peering" "default" { + count = var.network_peering == true ? 1 : 0 + name = "cdf-${var.name}" + network = "projects/${var.project_id}/global/networks/${var.network}" + peer_network = "projects/${local.tenant_project}/global/networks/${var.region}-${google_data_fusion_instance.default.name}" + export_custom_routes = true + import_custom_routes = true +} + +resource "google_compute_firewall" "default" { + count = var.firewall_create == true ? 1 : 0 + name = "${var.name}-allow-ssh" + project = var.project_id + network = var.network + source_ranges = [local.ip_allocation] + target_tags = ["${var.name}-allow-ssh"] + + allow { + protocol = "tcp" + ports = ["22"] + } +} + +resource "google_data_fusion_instance" "default" { + provider = google-beta + project = var.project_id + name = var.name + type = var.type + description = var.description + labels = var.labels + region = var.region + private_instance = var.private_instance + enable_stackdriver_logging = var.enable_stackdriver_logging + enable_stackdriver_monitoring = var.enable_stackdriver_monitoring + network_config { + network = var.network + ip_allocation = local.ip_allocation + } +} + diff --git a/modules/datafusion/outputs.tf b/modules/datafusion/outputs.tf new file mode 100644 index 00000000..92adf5e4 --- /dev/null +++ b/modules/datafusion/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "DataFusion instance ID." + value = google_data_fusion_instance.default.id +} + +output "ip_allocation" { + description = "IP range reserved for Data Fusion instance in case of a private instance." + value = "${local.ip_allocation}" +} + +output "resource" { + description = "DataFusion resource." + value = google_data_fusion_instance.default +} + +output "service_account" { + description = "DataFusion Service Account." + value = google_data_fusion_instance.default.service_account +} + +output "service_endpoint" { + description = "DataFusion Service Endpoint." + value = google_data_fusion_instance.default.service_endpoint +} + +output "version" { + description = "DataFusion version." + value = google_data_fusion_instance.default.version +} diff --git a/modules/datafusion/variables.tf b/modules/datafusion/variables.tf new file mode 100644 index 00000000..f10a6e3f --- /dev/null +++ b/modules/datafusion/variables.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +############################################################################### +# DtaFusion variables # +############################################################################### + +variable "description" { + description = "DataFuzion instance description." + type = string + default = "Terraform managed." +} + +variable "enable_stackdriver_logging" { + description = "Option to enable Stackdriver Logging." + type = bool + default = false +} + +variable "enable_stackdriver_monitoring" { + description = "Option to enable Stackdriver Monitorig." + type = bool + default = false +} + +variable "labels" { + description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs." + type = map(string) + default = {} +} + +variable "name" { + description = "Name of the DataFusion instance." + type = string +} + +variable "network" { + description = "Name of the network in the project with which the tenant project will be peered for executing pipelines in the form of projects/{project-id}/global/networks/{network}" + type = string +} + +variable "firewall_create" { + description = "Create Network firewall rules to enable SSH." + type = bool + default = true +} + +variable "network_peering" { + description = "Create Network peering between project and DataFusion tenant project." + type = bool + default = true +} + +variable "private_instance" { + description = "Create private instance." + type = bool + default = true +} + +variable "project_id" { + description = "Project ID." + type = string +} + +variable "region" { + description = "DataFusion region." + type = string +} + +variable "ip_allocation_create" { + description = "Create Ip range for datafusion instance." + type = bool + default = true +} + +variable "ip_allocation" { + description = "Ip allocated for datafusion instance when not using the auto created one and created outside of the module." + type = string + default = null +} + +variable "type" { + description = "Datafusion Instance type. It can be BASIC or ENTERPRISE (default value)." + type = string + default = "ENTERPRISE" +} diff --git a/modules/datafusion/versions.tf b/modules/datafusion/versions.tf new file mode 100644 index 00000000..bc4c2a9d --- /dev/null +++ b/modules/datafusion/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/dns/README.md b/modules/dns/README.md index 6993e569..d391c4cf 100644 --- a/modules/dns/README.md +++ b/modules/dns/README.md @@ -1,6 +1,8 @@ # Google Cloud DNS Module -This module allows simple management of Google Cloud DNS zones and records. It supports creating public, private, forwarding, and peering zones. For DNSSEC configuration, refer to the [`dns_managed_zone` documentation](https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config). +This module allows simple management of Google Cloud DNS zones and records. It supports creating public, private, forwarding, peering and service directory based zones. + +For DNSSEC configuration, refer to the [`dns_managed_zone` documentation](https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config). ## Example @@ -32,14 +34,16 @@ module "private-dns" { | *description* | Domain description. | string | | Terraform managed. | | *dnssec_config* | DNSSEC configuration: kind, non_existence, state. | any | | {} | | *forwarders* | List of target name servers, only valid for 'forwarding' zone types. | list(string) | | [] | -| *peer_network* | Peering network self link, only valid for 'peering' zone types. | string | | | +| *peer_network* | Peering network self link, only valid for 'peering' zone types. | string | | null | | *recordsets* | List of DNS record objects to manage. | list(object({...})) | | [] | -| *type* | Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'. | string | | private | +| *service_directory_namespace* | Service directory namespace id (URL), only valid for 'service-directory' zone types. | string | | null | +| *type* | Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering', 'service-directory'. | string | | private | ## Outputs | name | description | sensitive | |---|---|:---:| +| dns_keys | DNSKEY and DS records of DNSSEC-signed managed zones. | | | domain | The DNS zone domain. | | | name | The DNS zone name. | | | name_servers | The DNS zone name servers. | | diff --git a/modules/dns/main.tf b/modules/dns/main.tf index 6e098b8d..f29d2768 100644 --- a/modules/dns/main.tf +++ b/modules/dns/main.tf @@ -15,7 +15,6 @@ */ locals { - is_static_zone = var.type == "public" || var.type == "private" recordsets = var.recordsets == null ? {} : { for record in var.recordsets : join("/", [record.name, record.type]) => record @@ -25,6 +24,9 @@ locals { google_dns_managed_zone.public.0, null ) ) + dns_keys = try( + data.google_dns_keys.dns_keys.0, null + ) } resource "google_dns_managed_zone" "non-public" { @@ -38,14 +40,11 @@ resource "google_dns_managed_zone" "non-public" { dynamic forwarding_config { for_each = ( - var.type == "forwarding" && var.forwarders != null - ? { config = var.forwarders } - : {} + var.type == "forwarding" && var.forwarders != null ? [""] : [] ) - iterator = config content { dynamic "target_name_servers" { - for_each = config.value + for_each = var.forwarders iterator = address content { ipv4_address = address.value @@ -56,14 +55,11 @@ resource "google_dns_managed_zone" "non-public" { dynamic peering_config { for_each = ( - var.type == "peering" && var.peer_network != null - ? { config = var.peer_network } - : {} + var.type == "peering" && var.peer_network != null ? [""] : [] ) - iterator = config content { target_network { - network_url = config.value + network_url = var.peer_network } } } @@ -78,6 +74,19 @@ resource "google_dns_managed_zone" "non-public" { } } + dynamic service_directory_config { + for_each = ( + var.type == "service-directory" && var.service_directory_namespace != null + ? [""] + : [] + ) + content { + namespace { + namespace_url = var.service_directory_namespace + } + } + } + } resource "google_dns_managed_zone" "public" { @@ -113,6 +122,11 @@ resource "google_dns_managed_zone" "public" { } +data "google_dns_keys" "dns_keys" { + count = var.dnssec_config == {} || var.type != "public" ? 0 : 1 + managed_zone = google_dns_managed_zone.public.0.id +} + resource "google_dns_record_set" "cloud-static-records" { for_each = ( var.type == "public" || var.type == "private" diff --git a/modules/dns/outputs.tf b/modules/dns/outputs.tf index 87add740..ebb5f766 100644 --- a/modules/dns/outputs.tf +++ b/modules/dns/outputs.tf @@ -38,3 +38,8 @@ output "name_servers" { description = "The DNS zone name servers." value = try(local.zone.name_servers, null) } + +output "dns_keys" { + description = "DNSKEY and DS records of DNSSEC-signed managed zones." + value = local.dns_keys +} diff --git a/modules/dns/variables.tf b/modules/dns/variables.tf index 0991038c..f38fb36a 100644 --- a/modules/dns/variables.tf +++ b/modules/dns/variables.tf @@ -30,9 +30,6 @@ variable "description" { default = "Terraform managed." } -# TODO(ludoo): add link to DNSSEC documentation in README -# https://www.terraform.io/docs/providers/google/r/dns_managed_zone.html#dnssec_config - variable "default_key_specs_key" { description = "DNSSEC default key signing specifications: algorithm, key_length, key_type, kind." type = any @@ -71,7 +68,7 @@ variable "name" { variable "peer_network" { description = "Peering network self link, only valid for 'peering' zone types." type = string - default = "" + default = null } variable "project_id" { @@ -90,8 +87,14 @@ variable "recordsets" { default = [] } +variable "service_directory_namespace" { + description = "Service directory namespace id (URL), only valid for 'service-directory' zone types." + type = string + default = null +} + variable "type" { - description = "Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering'." + description = "Type of zone to create, valid values are 'public', 'private', 'forwarding', 'peering', 'service-directory'." type = string default = "private" } diff --git a/modules/dns/versions.tf b/modules/dns/versions.tf index ce6918e0..50c4c4e5 100644 --- a/modules/dns/versions.tf +++ b/modules/dns/versions.tf @@ -15,5 +15,9 @@ */ terraform { - required_version = ">= 0.12.6" + required_version = ">= 0.12.20" + required_providers { + google = "~> 3.10" + google-beta = "~> 3.20" + } } diff --git a/modules/endpoints/README.md b/modules/endpoints/README.md new file mode 100644 index 00000000..f57952e4 --- /dev/null +++ b/modules/endpoints/README.md @@ -0,0 +1,44 @@ +# Google Cloud Endpoints + +This module allows simple management of ['Google Cloud Endpoints'](https://cloud.google.com/endpoints/) services. It supports creating ['OpenAPI'](https://cloud.google.com/endpoints/docs/openapi) or ['gRPC'](https://cloud.google.com/endpoints/docs/grpc/about-grpc) endpoints. + +## Examples + +### OpenAPI + +```hcl +module "endpoint" { + source = "../../modules/endpoint" + project_id = "my-project" + service_name = "YOUR-API.endpoints.YOUR-PROJECT-ID.cloud.goog" + openapi_config = { "yaml_path" = "openapi.yaml" } + grpc_config = null + iam_roles = ["servicemanagement.serviceController"] + iam_members = { + "servicemanagement.serviceController" = ["serviceAccount:PROJECT_NUMBER-compute@developer.gserviceaccount.com"] + } +} +``` + +[Here](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/endpoints/getting-started/openapi.yaml) you can find an example of an openapi.yaml file. Once created the endpoint, remember to activate the service at project level. + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| grpc_config | The configuration for a gRPC enpoint. Either this or openapi_config must be specified. | object({...}) | ✓ | | +| openapi_config | The configuration for an OpenAPI endopoint. Either this or grpc_config must be specified. | object({...}) | ✓ | | +| service_name | The name of the service. Usually of the form '$apiname.endpoints.$projectid.cloud.goog'. | string | ✓ | | +| *iam_members* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved. | map(list(string)) | | {} | +| *iam_roles* | Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. | list(string) | | [] | +| *project_id* | The project ID that the service belongs to. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| endpoints | A list of Endpoint objects. | | +| endpoints_service | The Endpoint service resource. | | +| service_name | The name of the service.. | | + diff --git a/modules/endpoints/main.tf b/modules/endpoints/main.tf new file mode 100644 index 00000000..1b9cedbe --- /dev/null +++ b/modules/endpoints/main.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + iam_roles_bindings = { + for k in var.iam_roles : k => lookup(var.iam_members, k, []) + } +} + +resource "google_endpoints_service" "default" { + project = var.project_id + service_name = var.service_name + openapi_config = var.openapi_config != null ? file(var.openapi_config.yaml_path) : null + grpc_config = var.grpc_config != null ? file(var.grpc_config.yaml_path) : null + protoc_output_base64 = var.grpc_config != null ? base64encode(file(var.grpc_config.protoc_output_path)) : null +} + +resource "google_endpoints_service_iam_binding" "default" { + for_each = local.iam_roles_bindings + service_name = google_endpoints_service.default.service_name + role = "roles/${each.key}" + members = each.value +} diff --git a/modules/endpoints/outputs.tf b/modules/endpoints/outputs.tf new file mode 100644 index 00000000..e4da1de6 --- /dev/null +++ b/modules/endpoints/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_name" { + description = "The name of the service.." + value = google_endpoints_service.default.service_name +} + +output "endpoints_service" { + description = "The Endpoint service resource." + value = google_endpoints_service.default +} + +output "endpoints" { + description = "A list of Endpoint objects." + value = google_endpoints_service.default.endpoints +} diff --git a/modules/endpoints/variables.tf b/modules/endpoints/variables.tf new file mode 100644 index 00000000..76fb8b8b --- /dev/null +++ b/modules/endpoints/variables.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "grpc_config" { + description = "The configuration for a gRPC enpoint. Either this or openapi_config must be specified." + type = object({ + yaml_path = string + protoc_output_path = string + }) +} + +variable "iam_roles" { + description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members." + type = list(string) + default = [] +} + +variable "iam_members" { + description = "Authoritative for a given role. Updates the IAM policy to grant a role to a list of members. Other roles within the IAM policy for the instance are preserved." + type = map(list(string)) + default = {} +} + +variable "openapi_config" { + description = "The configuration for an OpenAPI endopoint. Either this or grpc_config must be specified." + type = object({ + yaml_path = string + }) +} + +variable "project_id" { + description = "The project ID that the service belongs to." + type = string + default = null +} + +variable "service_name" { + description = "The name of the service. Usually of the form '$apiname.endpoints.$projectid.cloud.goog'." + type = string +} diff --git a/modules/endpoints/versions.tf b/modules/endpoints/versions.tf new file mode 100644 index 00000000..bc4c2a9d --- /dev/null +++ b/modules/endpoints/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/folders/README.md b/modules/folders/README.md index b1568711..7efb4b68 100644 --- a/modules/folders/README.md +++ b/modules/folders/README.md @@ -8,12 +8,12 @@ This module allow creation and management of sets of folders sharing a common pa ```hcl module "folder" { - source = "./modules/folder" + source = "./modules/folders" parent = "organizations/1234567890" - names = ["Folder one", "Folder two] + names = ["Folder one", "Folder two"] iam_members = { "Folder one" = { - "roles/owner" => ["group:users@example.com"] + "roles/owner" = ["group:users@example.com"] } } iam_roles = { @@ -26,9 +26,9 @@ module "folder" { ```hcl module "folder" { - source = "./modules/folder" + source = "./modules/folders" parent = "organizations/1234567890" - names = ["Folder one", "Folder two] + names = ["Folder one", "Folder two"] policy_boolean = { "constraints/compute.disableGuestAttributesAccess" = true "constraints/compute.skipDefaultNetworkCreation" = true diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 98545db8..839faadd 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -26,6 +26,31 @@ module "buckets" { } ``` +### Example with Cloud KMS + +```hcl +module "buckets" { + source = "./modules/gcs" + project_id = "myproject" + prefix = "test" + names = ["bucket-one", "bucket-two"] + bucket_policy_only = { + bucket-one = false + } + iam_members = { + bucket-two = { + "roles/storage.admin" = ["group:storage@example.com"] + } + } + iam_roles = { + bucket-two = ["roles/storage.admin"] + } + kms_keys = { + bucket-two = local.kms_key.self_link, + } +} +``` + ## Variables @@ -34,12 +59,13 @@ module "buckets" { | names | Bucket name suffixes. | list(string) | ✓ | | | project_id | Bucket project id. | string | ✓ | | | *bucket_policy_only* | Optional map to disable object ACLS keyed by name, defaults to true. | map(bool) | | {} | +| *encryption_keys* | Per-bucket KMS keys that will be used for encryption. | map(string) | | {} | | *force_destroy* | Optional map to set force destroy keyed by name, defaults to false. | map(bool) | | {} | -| *iam_members* | IAM members keyed by bucket name and role. | map(map(list(string))) | | null | -| *iam_roles* | IAM roles keyed by bucket name. | map(list(string)) | | null | +| *iam_members* | IAM members keyed by bucket name and role. | map(map(list(string))) | | {} | +| *iam_roles* | IAM roles keyed by bucket name. | map(list(string)) | | {} | | *labels* | Labels to be attached to all buckets. | map(string) | | {} | | *location* | Bucket location. | string | | EU | -| *prefix* | Prefix used to generate the bucket name. | string | | | +| *prefix* | Prefix used to generate the bucket name. | string | | null | | *storage_class* | Bucket storage class. | string | | MULTI_REGIONAL | | *versioning* | Optional map to set versioning keyed by name, defaults to false. | map(bool) | | {} | diff --git a/modules/gcs/main.tf b/modules/gcs/main.tf index baff11d9..f345ed3b 100644 --- a/modules/gcs/main.tf +++ b/modules/gcs/main.tf @@ -31,7 +31,12 @@ locals { "${pair.name}-${pair.role}" => pair } iam_members = var.iam_members == null ? {} : var.iam_members - prefix = var.prefix == "" ? "" : join("-", [var.prefix, lower(var.location), ""]) + prefix = ( + var.prefix == null || var.prefix == "" # keep "" for backward compatibility + ? "" + : join("-", [var.prefix, lower(var.location), ""]) + ) + kms_keys = { for name in var.names : name => lookup(var.encryption_keys, name, null) } } resource "google_storage_bucket" "buckets" { @@ -50,6 +55,14 @@ resource "google_storage_bucket" "buckets" { name = lower(each.key) storage_class = lower(var.storage_class) }) + + dynamic encryption { + for_each = local.kms_keys[each.key] == null ? [] : [""] + + content { + default_kms_key_name = local.kms_keys[each.key] + } + } } resource "google_storage_bucket_iam_binding" "bindings" { diff --git a/modules/gcs/variables.tf b/modules/gcs/variables.tf index 6cc712eb..cdc63ecc 100644 --- a/modules/gcs/variables.tf +++ b/modules/gcs/variables.tf @@ -29,13 +29,19 @@ variable "force_destroy" { variable "iam_members" { description = "IAM members keyed by bucket name and role." type = map(map(list(string))) - default = null + default = {} } variable "iam_roles" { description = "IAM roles keyed by bucket name." type = map(list(string)) - default = null + default = {} +} + +variable "encryption_keys" { + description = "Per-bucket KMS keys that will be used for encryption." + type = map(string) + default = {} } variable "labels" { @@ -58,7 +64,7 @@ variable "names" { variable "prefix" { description = "Prefix used to generate the bucket name." type = string - default = "" + default = null } variable "project_id" { diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index fcbae58f..7442ae53 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -41,7 +41,7 @@ module "cluster-1" { | secondary_range_pods | Subnet secondary range name used for pods. | string | ✓ | | | secondary_range_services | Subnet secondary range name used for services. | string | ✓ | | | subnetwork | VPC subnetwork name or self link. | string | ✓ | | -| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | ... | +| *addons* | Addons enabled in the cluster (true means enabled). | object({...}) | | ... | | *authenticator_security_group* | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null | | *cluster_autoscaling* | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({...}) | | ... | | *database_encryption* | Enable and configure GKE application-layer secrets encryption. | object({...}) | | ... | @@ -70,6 +70,7 @@ module "cluster-1" { | name | description | sensitive | |---|---|:---:| +| ca_certificate | Public certificate of the cluster (base64-encoded). | ✓ | | cluster | Cluster resource. | ✓ | | endpoint | Cluster endpoint. | | | location | Cluster location. | | diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf index 31ef293a..86fe095a 100644 --- a/modules/gke-cluster/main.tf +++ b/modules/gke-cluster/main.tf @@ -75,6 +75,9 @@ resource "google_container_cluster" "cluster" { disabled = ! var.addons.istio_config.enabled auth = var.addons.istio_config.tls ? "AUTH_MUTUAL_TLS" : "AUTH_NONE" } + gce_persistent_disk_csi_driver_config { + enabled = var.addons.gce_persistent_disk_csi_driver_config + } } # TODO(ludomagno): support setting address ranges instead of range names @@ -147,14 +150,15 @@ resource "google_container_cluster" "cluster" { enabled = true resource_limits { resource_type = "cpu" - minimum = config.cpu_min - maximum = config.cpu_max + minimum = config.value.cpu_min + maximum = config.value.cpu_max } resource_limits { resource_type = "memory" - minimum = config.memory_min - maximum = config.memory_max + minimum = config.value.memory_min + maximum = config.value.memory_max } + // TODO: support GPUs too } } diff --git a/modules/gke-cluster/outputs.tf b/modules/gke-cluster/outputs.tf index 6937ee96..cb19e92d 100644 --- a/modules/gke-cluster/outputs.tf +++ b/modules/gke-cluster/outputs.tf @@ -39,3 +39,9 @@ output "name" { description = "Cluster name." value = google_container_cluster.cluster.name } + +output "ca_certificate" { + description = "Public certificate of the cluster (base64-encoded)." + value = google_container_cluster.cluster.master_auth.0.cluster_ca_certificate + sensitive = true +} diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf index 270a20f2..62b3709c 100644 --- a/modules/gke-cluster/variables.tf +++ b/modules/gke-cluster/variables.tf @@ -26,6 +26,7 @@ variable "addons" { tls = bool }) network_policy_config = bool + gce_persistent_disk_csi_driver_config = bool }) default = { cloudrun_config = false @@ -37,6 +38,7 @@ variable "addons" { tls = false } network_policy_config = false + gce_persistent_disk_csi_driver_config = false } } diff --git a/modules/iam-service-accounts/README.md b/modules/iam-service-accounts/README.md index 00be1075..7fafafc9 100644 --- a/modules/iam-service-accounts/README.md +++ b/modules/iam-service-accounts/README.md @@ -40,7 +40,7 @@ module "myproject-default-service-accounts" { | *iam_roles* | List of authoritative roles granted on the service accounts. | list(string) | | [] | | *iam_storage_roles* | Storage roles granted to all service accounts, by bucket name. | map(list(string)) | | {} | | *names* | Names of the service accounts to create. | list(string) | | [] | -| *prefix* | Prefix applied to service account names. | string | | | +| *prefix* | Prefix applied to service account names. | string | | null | ## Outputs diff --git a/modules/iam-service-accounts/main.tf b/modules/iam-service-accounts/main.tf index 21524fa3..12f6e128 100644 --- a/modules/iam-service-accounts/main.tf +++ b/modules/iam-service-accounts/main.tf @@ -54,32 +54,13 @@ locals { ] ] ]) - keys = ( - var.generate_keys - ? { - for name in var.names : - name => lookup(google_service_account_key.keys, name, null) - } - : {} - ) - prefix = ( - var.prefix != "" - ? "${var.prefix}-" - : "" - ) - resource = ( - length(var.names) > 0 - ? lookup(local.resources, var.names[0], null) - : null - ) + keys = var.generate_keys ? google_service_account_key.keys : {} + prefix = var.prefix != null ? "${var.prefix}-" : "" + resource = try(google_service_account.service_accounts[var.names[0]], null) resource_iam_emails = { - for name, resource in local.resources : + for name, resource in google_service_account.service_accounts : name => "serviceAccount:${resource.email}" } - resources = { - for name in var.names : - name => lookup(google_service_account.service_accounts, name, null) - } } resource "google_service_account" "service_accounts" { diff --git a/modules/iam-service-accounts/outputs.tf b/modules/iam-service-accounts/outputs.tf index 9901675e..530ccd1c 100644 --- a/modules/iam-service-accounts/outputs.tf +++ b/modules/iam-service-accounts/outputs.tf @@ -21,27 +21,30 @@ output "service_account" { output "service_accounts" { description = "Service account resources." - value = local.resources + value = google_service_account.service_accounts } output "email" { description = "Service account email (for single use)." - value = local.resource == null ? null : local.resource.email + value = try(local.resource.email, null) } output "iam_email" { description = "IAM-format service account email (for single use)." - value = local.resource == null ? null : "serviceAccount:${local.resource.email}" + value = try("serviceAccount:${local.resource.email}", null) } output "key" { description = "Service account key (for single use)." - value = lookup(local.keys, var.names[0], null) + value = try(local.keys[var.names[0]], null) } output "emails" { description = "Service account emails." - value = { for name, resource in local.resources : name => resource.email } + value = { + for name, resource in google_service_account.service_accounts : + name => resource.email + } } output "iam_emails" { @@ -51,12 +54,18 @@ output "iam_emails" { output "emails_list" { description = "Service account emails." - value = [for name, resource in local.resources : resource.email] + value = [ + for name, resource in google_service_account.service_accounts : + resource.email + ] } output "iam_emails_list" { description = "IAM-format service account emails." - value = [for name, resource in local.resources : "serviceAccount:${resource.email}"] + value = [ + for name, resource in google_service_account.service_accounts : + "serviceAccount:${resource.email}" + ] } output "keys" { diff --git a/modules/iam-service-accounts/variables.tf b/modules/iam-service-accounts/variables.tf index 60e0ed5f..adcdc671 100644 --- a/modules/iam-service-accounts/variables.tf +++ b/modules/iam-service-accounts/variables.tf @@ -20,23 +20,6 @@ variable "generate_keys" { default = false } -variable "names" { - description = "Names of the service accounts to create." - type = list(string) - default = [] -} - -variable "prefix" { - description = "Prefix applied to service account names." - type = string - default = "" -} - -variable "project_id" { - description = "Project id where service account will be created." - type = string -} - variable "iam_members" { description = "Map of member lists which are granted authoritative roles on the service accounts, keyed by role." type = map(list(string)) @@ -78,3 +61,20 @@ variable "iam_storage_roles" { type = map(list(string)) default = {} } + +variable "names" { + description = "Names of the service accounts to create." + type = list(string) + default = [] +} + +variable "prefix" { + description = "Prefix applied to service account names." + type = string + default = null +} + +variable "project_id" { + description = "Project id where service account will be created." + type = string +} diff --git a/modules/kms/README.md b/modules/kms/README.md index 55a0e944..69ceb882 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -1,40 +1,71 @@ # Google KMS Module -Simple Cloud KMS module that allows managing a keyring, zero or more keys in the keyring, and IAM role bindings on individual keys. +This module allows creating and managing KMS crypto keys and IAM bindings at both the keyring and crypto key level. An existing keyring can be used, or a new one can be created and managed by the module if needed. -The `protected` flag in the `key_attributes` variable sets the `prevent_destroy` lifecycle argument on an a per-key basis. +When using an existing keyring be mindful about applying IAM bindings, as all bindings used by this module are authoritative, and you might inadvertently override bindings managed by the keyring creator. + +## Protecting against destroy + +In this module **no lifecycle blocks are set on resources to prevent destroy**, in order to allow for experimentation and testing where rapid `apply`/`destroy` cycles are needed. If you plan on using this module to manage non-development resources, **clone it and uncomment the lifecycle blocks** found in `main.tf`. ## Examples -### Minimal example +### Using an existing keyring ```hcl module "kms" { - source = "../modules/kms" - project_id = "my-project" - keyring = "test" - location = "europe" - keys = ["key-a", "key-b"] + source = "../modules/kms" + project_id = "my-project" + iam_roles = ["roles/owner"] + iam_members = { + "roles/owner" = ["user:user1@example.com"] + } + keyring = { location = "europe-west1", name = "test" } + keyring_create = false + keys = { key-a = null, key-b = null, key-c = null } } ``` -### Granting access to keys via IAM +### Keyring creation and crypto key rotation and IAM roles ```hcl module "kms" { - source = "../modules/kms" - project_id = "my-project" - keyring = "test" - location = "europe" - keys = ["key-a", "key-b"] - iam_roles = { - key-a = ["roles/cloudkms.cryptoKeyDecrypter"] + source = "../modules/kms" + project_id = "my-project" + key_iam_roles = { + key-a = ["roles/owner"] } - iam_members = { + key_iam_members = { key-a = { - "roles/cloudkms.cryptoKeyDecrypter" = ["user:me@example.org"] + "roles/owner" = ["user:user1@example.com"] } } + keyring = { location = "europe-west1", name = "test" } + keys = { + key-a = null + key-b = { rotation_period = "604800s", labels = null } + key-c = { rotation_period = null, labels = { env = "test" } } + } +} +``` + +### Crypto key purpose + +```hcl +module "kms" { + source = "../modules/kms" + project_id = "my-project" + key_purpose = { + key-c = { + purpose = "ASYMMETRIC_SIGN" + version_template = { + algorithm = "EC_SIGN_P384_SHA384" + protection_level = null + } + } + } + keyring = { location = "europe-west1", name = "test" } + keys = { key-a = null, key-b = null, key-c = null } } ``` @@ -43,14 +74,16 @@ module "kms" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| -| keyring | Keyring name. | string | ✓ | | -| location | Location for the keyring. | string | ✓ | | +| keyring | Keyring attributes. | object({...}) | ✓ | | | project_id | Project id where the keyring will be created. | string | ✓ | | -| *iam_members* | IAM members keyed by key name and role. | map(map(list(string))) | | {} | -| *iam_roles* | IAM roles keyed by key name. | map(list(string)) | | {} | -| *key_attributes* | Optional key attributes per key. | map(object({...})) | | {} | -| *key_defaults* | Key attribute defaults. | object({...}) | | ... | -| *keys* | Key names. | list(string) | | [] | +| *iam_members* | Keyring IAM members. | map(list(string)) | | {} | +| *iam_roles* | Keyring IAM roles. | list(string) | | [] | +| *key_iam_members* | IAM members keyed by key name and role. | map(map(list(string))) | | {} | +| *key_iam_roles* | IAM roles keyed by key name. | map(list(string)) | | {} | +| *key_purpose* | Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | map(object({...})) | | {} | +| *key_purpose_defaults* | Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required. | object({...}) | | ... | +| *keyring_create* | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | +| *keys* | Key names and base attributes. Set attributes to null if not needed. | map(object({...})) | | {} | ## Outputs @@ -59,25 +92,7 @@ module "kms" { | key_self_links | Key self links. | | | keyring | Keyring resource. | | | keys | Key resources. | | -| location | Keyring self link. | | -| name | Keyring self link. | | +| location | Keyring location. | | +| name | Keyring name. | | | self_link | Keyring self link. | | - -## Requirements - -These sections describe requirements for using this module. - -### IAM - -The following roles must be used to provision the resources of this module: - -- Cloud KMS Admin: `roles/cloudkms.admin` or -- Owner: `roles/owner` - -### APIs - -A project with the following APIs enabled must be used to host the -resources of this module: - -- Google Cloud Key Management Service: `cloudkms.googleapis.com` diff --git a/modules/kms/main.tf b/modules/kms/main.tf index 18c14eff..b488ac71 100644 --- a/modules/kms/main.tf +++ b/modules/kms/main.tf @@ -15,59 +15,74 @@ */ locals { - # distinct is needed to make the expanding function argument work - iam_pairs = flatten([ - for name, roles in var.iam_roles : + key_iam_pairs = flatten([ + for name, roles in var.key_iam_roles : [for role in roles : { name = name, role = role }] ]) - iam_keypairs = { - for pair in local.iam_pairs : + key_iam_keypairs = { + for pair in local.key_iam_pairs : "${pair.name}-${pair.role}" => pair } - key_attributes = { - for name in var.keys : - name => lookup(var.key_attributes, name, var.key_defaults) + key_purpose = { + for key, attrs in var.keys : key => try( + var.key_purpose[key], var.key_purpose_defaults + ) } - keys = merge( - { for name, resource in google_kms_crypto_key.keys : name => resource }, - { for name, resource in google_kms_crypto_key.keys-ephemeral : name => resource } + keyring = ( + var.keyring_create + ? google_kms_key_ring.default.0 + : data.google_kms_key_ring.default.0 ) } -resource "google_kms_key_ring" "key_ring" { - name = var.keyring +data "google_kms_key_ring" "default" { + count = var.keyring_create ? 0 : 1 project = var.project_id - location = var.location + name = var.keyring.name + location = var.keyring.location } -resource "google_kms_crypto_key" "keys" { - for_each = { - for name, attrs in local.key_attributes : - name => attrs if attrs.protected - } +resource "google_kms_key_ring" "default" { + count = var.keyring_create ? 1 : 0 + project = var.project_id + name = var.keyring.name + location = var.keyring.location + # lifecycle { + # prevent_destroy = true + # } +} + +resource "google_kms_key_ring_iam_binding" "default" { + for_each = toset(var.iam_roles) + key_ring_id = local.keyring.self_link + role = each.value + members = lookup(var.iam_members, each.value, []) +} + +resource "google_kms_crypto_key" "default" { + for_each = var.keys + key_ring = local.keyring.self_link name = each.key - key_ring = google_kms_key_ring.key_ring.self_link - rotation_period = each.value.rotation_period - lifecycle { - prevent_destroy = true + rotation_period = try(each.value.rotation_period, null) + labels = try(each.value.labels, null) + purpose = try(local.key_purpose[each.key].purpose, null) + dynamic version_template { + for_each = local.key_purpose[each.key].version_template == null ? [] : [""] + content { + algorithm = local.key_purpose[each.key].version_template.algorithm + protection_level = local.key_purpose[each.key].version_template.protection_level + } } + # lifecycle { + # prevent_destroy = true + # } } -resource "google_kms_crypto_key" "keys-ephemeral" { - for_each = { - for name, attrs in local.key_attributes : - name => attrs if ! attrs.protected - } - name = each.key - key_ring = google_kms_key_ring.key_ring.self_link - rotation_period = each.value.rotation_period -} - -resource "google_kms_crypto_key_iam_binding" "bindings" { - for_each = local.iam_keypairs +resource "google_kms_crypto_key_iam_binding" "default" { + for_each = local.key_iam_keypairs role = each.value.role - crypto_key_id = local.keys[each.value.name].self_link + crypto_key_id = google_kms_crypto_key.default[each.value.name].self_link members = lookup( - lookup(var.iam_members, each.value.name, {}), each.value.role, [] + lookup(var.key_iam_members, each.value.name, {}), each.value.role, [] ) } diff --git a/modules/kms/outputs.tf b/modules/kms/outputs.tf index de30ea17..d92190a5 100644 --- a/modules/kms/outputs.tf +++ b/modules/kms/outputs.tf @@ -16,30 +16,51 @@ output "keyring" { description = "Keyring resource." - value = google_kms_key_ring.key_ring + value = local.keyring + depends_on = [ + google_kms_key_ring_iam_binding.default + ] } output "location" { - description = "Keyring self link." - value = google_kms_key_ring.key_ring.location + description = "Keyring location." + value = local.keyring.location + depends_on = [ + google_kms_key_ring_iam_binding.default + ] } output "name" { - description = "Keyring self link." - value = google_kms_key_ring.key_ring.name + description = "Keyring name." + value = local.keyring.name + depends_on = [ + google_kms_key_ring_iam_binding.default + ] } output "self_link" { description = "Keyring self link." - value = google_kms_key_ring.key_ring.self_link + value = local.keyring.self_link + depends_on = [ + google_kms_key_ring_iam_binding.default + ] } output "keys" { description = "Key resources." - value = local.keys + value = google_kms_crypto_key.default + depends_on = [ + google_kms_crypto_key_iam_binding.default + ] } output "key_self_links" { description = "Key self links." - value = { for name, resource in local.keys : name => resource.self_link } + value = { + for name, resource in google_kms_crypto_key.default : + name => resource.self_link + } + depends_on = [ + google_kms_crypto_key_iam_binding.default + ] } diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf index e976b72b..42ec689b 100644 --- a/modules/kms/variables.tf +++ b/modules/kms/variables.tf @@ -15,53 +15,79 @@ */ variable "iam_members" { + description = "Keyring IAM members." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "Keyring IAM roles." + type = list(string) + default = [] +} + +variable "key_iam_members" { description = "IAM members keyed by key name and role." type = map(map(list(string))) default = {} } -variable "iam_roles" { +variable "key_iam_roles" { description = "IAM roles keyed by key name." type = map(list(string)) default = {} } -variable "keyring" { - description = "Keyring name." - type = string -} - -variable "key_attributes" { - description = "Optional key attributes per key." +variable "key_purpose" { + description = "Per-key purpose, if not set defaults will be used. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required." type = map(object({ - protected = bool - rotation_period = string + purpose = string + version_template = object({ + algorithm = string + protection_level = string + }) })) default = {} } -variable "key_defaults" { - description = "Key attribute defaults." +variable "key_purpose_defaults" { + description = "Defaults used for key purpose when not defined at the key level. If purpose is not `ENCRYPT_DECRYPT` (the default), `version_template.algorithm` is required." type = object({ - protected = bool - rotation_period = string + purpose = string + version_template = object({ + algorithm = string + protection_level = string + }) }) default = { - protected = true - rotation_period = "100000s" + purpose = null + version_template = null } } -variable "keys" { - description = "Key names." - type = list(string) - default = [] +# cf https://cloud.google.com/kms/docs/locations + +variable "keyring" { + description = "Keyring attributes." + type = object({ + location = string + name = string + }) } -# cf https://cloud.google.com/kms/docs/locations -variable "location" { - description = "Location for the keyring." - type = string +variable "keyring_create" { + description = "Set to false to manage keys and IAM bindings in an existing keyring." + type = bool + default = true +} + +variable "keys" { + description = "Key names and base attributes. Set attributes to null if not needed." + type = map(object({ + rotation_period = string + labels = map(string) + })) + default = {} } variable "project_id" { diff --git a/modules/net-cloudnat/README.md b/modules/net-cloudnat/README.md index b8bb17ce..89a9c352 100644 --- a/modules/net-cloudnat/README.md +++ b/modules/net-cloudnat/README.md @@ -28,8 +28,8 @@ module "nat" { | *config_timeouts* | Timeout configurations. | object({...}) | | ... | | *router_asn* | Router ASN used for auto-created router. | number | | 64514 | | *router_create* | Create router. | bool | | true | -| *router_name* | Router name, leave blank if router will be created to use auto generated name. | string | | | -| *router_network* | Name of the VPC used for auto-created router. | string | | | +| *router_name* | Router name, leave blank if router will be created to use auto generated name. | string | | null | +| *router_network* | Name of the VPC used for auto-created router. | string | | null | | *subnetworks* | Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS. | list(object({...})) | | [] | ## Outputs diff --git a/modules/net-ilb/README.md b/modules/net-ilb/README.md index 3669c54b..1d90c5aa 100644 --- a/modules/net-ilb/README.md +++ b/modules/net-ilb/README.md @@ -2,6 +2,10 @@ This module allows managing a GCE Internal Load Balancer and integrates the forwarding rule, regional backend, and optional health check resources. It's designed to be a simple match for the [`compute-vm`](../compute-vm) module, which can be used to manage instance templates and instance groups. +## TODO + +- [ ] add a variable for setting address purpose to `SHARED_LOADBALANCER_VIP` and an output for the address once the [provider support has been implemented](https://github.com/terraform-providers/terraform-provider-google/issues/6499) + ## Issues TODO(ludoo): check if this is still the case after splitting out MIG from compute-vm @@ -10,7 +14,40 @@ There are some corner cases (eg when switching the instance template from intern One other issue is a `Provider produced inconsistent final plan` error which is sometimes raised when switching template version. This seems to be related to this [open provider issue](https://github.com/terraform-providers/terraform-provider-google/issues/3937), but it's relatively harmless since the resource is updated, and subsequent applies raise no errors. -## Example +## Examples + +### Externally managed instances + +This examples shows how to create an ILB by combining externally managed instances (in a custom module or even outside of the current root module) in an unmanaged group. When using internally managed groups, remember to run `terraform apply` each time group instances change. + +```hcl +module "ilb" { + source = "./modules/net-ilb" + project_id = "my-project" + region = "europe-west1" + name = "ilb-test" + service_label = "ilb-test" + network = local.network_self_link + subnetwork = local.subnetwork_self_link + group_configs = { + my-group = { + zone = europe-west1-b, named_ports = null, instances = [ + local.instance1_self_link, local.instance2_self_link + ] + } + } + backends = [{ + failover = false + group = module.ilb.groups.my-group.self_link + balancing_mode = "CONNECTION" + }] + health_check_config = { + type = "http", check = { port = 80 }, config = {}, logging = true + } +} +``` + +### End to end example This example spins up a simple HTTP server and combines four modules: @@ -85,10 +122,10 @@ module "ilb" { | *backend_config* | Optional backend configuration. | object({...}) | | null | | *failover_config* | Optional failover configuration. | object({...}) | | null | | *global_access* | Global access, defaults to false if not set. | bool | | null | +| *group_configs* | Optional unmanaged groups to create. Can be referenced in backends via outputs. | map(object({...})) | | {} | | *health_check* | Name of existing health check to use, disables auto-created health check. | string | | null | -| *health_check_config* | Configuration of the auto-created helth check. | object({...}) | | ... | +| *health_check_config* | Configuration of the auto-created helth check. | object({...}) | | ... | | *labels* | Labels set on resources. | map(string) | | {} | -| *log_sample_rate* | Set a value between 0 and 1 to enable logging for resources, and set the sampling rate for backend logging. | number | | null | | *ports* | Comma-separated ports, leave null to use all ports. | list(string) | | null | | *protocol* | IP protocol used, defaults to TCP. | string | | TCP | | *service_label* | Optional prefix of the fully qualified forwarding rule name. | string | | null | @@ -104,6 +141,8 @@ module "ilb" { | forwarding_rule_address | Forwarding rule address. | | | forwarding_rule_id | Forwarding rule id. | | | forwarding_rule_self_link | Forwarding rule self link. | | +| group_self_links | Optional unmanaged instance group self links. | | +| groups | Optional unmanaged instance group resources. | | | health_check | Auto-created health-check resource. | | | health_check_self_id | Auto-created health-check self id. | | | health_check_self_link | Auto-created health-check self link. | | diff --git a/modules/net-ilb/main.tf b/modules/net-ilb/main.tf index eb6ee475..75440d2f 100644 --- a/modules/net-ilb/main.tf +++ b/modules/net-ilb/main.tf @@ -29,6 +29,7 @@ locals { google_compute_health_check.http2.0, {} ) + health_check_type = try(var.health_check_config.type, null) } resource "google_compute_forwarding_rule" "default" { @@ -89,9 +90,28 @@ resource "google_compute_region_backend_service" "default" { } +resource "google_compute_instance_group" "unmanaged" { + for_each = var.group_configs + project = var.project_id + zone = each.value.zone + name = each.key + description = "Terraform-managed." + instances = each.value.instances + dynamic named_port { + for_each = each.value.named_ports != null ? each.value.named_ports : {} + iterator = config + content { + name = config.key + port = config.value + } + } +} + resource "google_compute_health_check" "http" { - provider = google-beta - count = try(var.health_check_config.type, null) == "http" ? 1 : 0 + provider = google-beta + count = ( + var.health_check == null && local.health_check_type == "http" ? 1 : 0 + ) project = var.project_id name = var.name description = "Terraform managed." @@ -120,8 +140,10 @@ resource "google_compute_health_check" "http" { } resource "google_compute_health_check" "https" { - provider = google-beta - count = try(var.health_check_config.type, null) == "https" ? 1 : 0 + provider = google-beta + count = ( + var.health_check == null && local.health_check_type == "https" ? 1 : 0 + ) project = var.project_id name = var.name description = "Terraform managed." @@ -150,8 +172,10 @@ resource "google_compute_health_check" "https" { } resource "google_compute_health_check" "tcp" { - provider = google-beta - count = try(var.health_check_config.type, null) == "tcp" ? 1 : 0 + provider = google-beta + count = ( + var.health_check == null && local.health_check_type == "tcp" ? 1 : 0 + ) project = var.project_id name = var.name description = "Terraform managed." @@ -179,8 +203,10 @@ resource "google_compute_health_check" "tcp" { } resource "google_compute_health_check" "ssl" { - provider = google-beta - count = try(var.health_check_config.type, null) == "ssl" ? 1 : 0 + provider = google-beta + count = ( + var.health_check == null && local.health_check_type == "ssl" ? 1 : 0 + ) project = var.project_id name = var.name description = "Terraform managed." @@ -208,8 +234,10 @@ resource "google_compute_health_check" "ssl" { } resource "google_compute_health_check" "http2" { - provider = google-beta - count = try(var.health_check_config.type, null) == "http2" ? 1 : 0 + provider = google-beta + count = ( + var.health_check == null && local.health_check_type == "http2" ? 1 : 0 + ) project = var.project_id name = var.name description = "Terraform managed." diff --git a/modules/net-ilb/outputs.tf b/modules/net-ilb/outputs.tf index 8418e40a..6f8ddd60 100644 --- a/modules/net-ilb/outputs.tf +++ b/modules/net-ilb/outputs.tf @@ -49,6 +49,18 @@ output "forwarding_rule_self_link" { value = google_compute_forwarding_rule.default.self_link } +output "groups" { + description = "Optional unmanaged instance group resources." + value = google_compute_instance_group.unmanaged +} + +output "group_self_links" { + description = "Optional unmanaged instance group self links." + value = { + for k, v in google_compute_instance_group.unmanaged : k => v.self_link + } +} + output "health_check" { description = "Auto-created health-check resource." value = local.health_check_resource diff --git a/modules/net-ilb/variables.tf b/modules/net-ilb/variables.tf index df7e7ee8..f4f0036a 100644 --- a/modules/net-ilb/variables.tf +++ b/modules/net-ilb/variables.tf @@ -55,6 +55,16 @@ variable "global_access" { default = null } +variable "group_configs" { + description = "Optional unmanaged groups to create. Can be referenced in backends via outputs." + type = map(object({ + instances = list(string) + named_ports = map(number) + zone = string + })) + default = {} +} + variable "health_check" { description = "Name of existing health check to use, disables auto-created health check." type = string diff --git a/modules/net-vpc-firewall/main.tf b/modules/net-vpc-firewall/main.tf index 7252e3a8..49affdd7 100644 --- a/modules/net-vpc-firewall/main.tf +++ b/modules/net-vpc-firewall/main.tf @@ -34,9 +34,7 @@ resource "google_compute_firewall" "allow-admins" { network = var.network project = var.project_id source_ranges = var.admin_ranges - allow { protocol = "icmp" } - allow { protocol = "tcp" } - allow { protocol = "udp" } + allow { protocol = "all" } } ############################################################################### diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md index 2a3a17b6..334c7fbb 100644 --- a/modules/net-vpc/README.md +++ b/modules/net-vpc/README.md @@ -112,20 +112,21 @@ module "vpc-host" { | name | The name of the network being created | string | ✓ | | | project_id | The ID of the project where this VPC will be created | string | ✓ | | | *auto_create_subnetworks* | Set to true to create an auto mode subnet, defaults to custom mode. | bool | | false | +| *delete_default_routes_on_create* | Set to true to delete the default routes at creation time. | bool | | false | | *description* | An optional description of this resource (triggers recreation on change). | string | | Terraform-managed. | -| *iam_members* | List of IAM members keyed by subnet 'region/name' and role. | map(map(list(string))) | | null | -| *iam_roles* | List of IAM roles keyed by subnet 'region/name'. | map(list(string)) | | null | +| *iam_members* | List of IAM members keyed by subnet 'region/name' and role. | map(map(list(string))) | | {} | +| *iam_roles* | List of IAM roles keyed by subnet 'region/name'. | map(list(string)) | | {} | | *log_config_defaults* | Default configuration for flow logs when enabled. | object({...}) | | ... | -| *log_configs* | Map keyed by subnet 'region/name' of optional configurations for flow logs when enabled. | map(map(string)) | | null | +| *log_configs* | Map keyed by subnet 'region/name' of optional configurations for flow logs when enabled. | map(map(string)) | | {} | | *peering_config* | VPC peering configuration. | object({...}) | | null | -| *routes* | Network routes, keyed by name. | map(object({...})) | | null | +| *routes* | Network routes, keyed by name. | map(object({...})) | | {} | | *routing_mode* | The network routing mode (default 'GLOBAL') | string | | GLOBAL | | *shared_vpc_host* | Enable shared VPC for this project. | bool | | false | | *shared_vpc_service_projects* | Shared VPC service projects to register with this host | list(string) | | [] | | *subnet_descriptions* | Optional map of subnet descriptions, keyed by subnet 'region/name'. | map(string) | | {} | | *subnet_flow_logs* | Optional map of boolean to control flow logs (default is disabled), keyed by subnet 'region/name'. | map(bool) | | {} | | *subnet_private_access* | Optional map of boolean to control private Google access (default is enabled), keyed by subnet 'region/name'. | map(bool) | | {} | -| *subnets* | The list of subnets being created | map(object({...})) | | null | +| *subnets* | The list of subnets being created | list(object({...})) | | [] | ## Outputs @@ -136,10 +137,10 @@ module "vpc-host" { | network | Network resource. | | | project_id | Shared VPC host project id. | | | self_link | The URI of the VPC being created. | | -| subnet_ips | Map of subnet address ranges keyed by 'region/name'. | | -| subnet_regions | Map of subnet regions keyed by 'region/name'. | | -| subnet_secondary_ranges | Map of subnet secondary ranges keyed by 'region/name'. | | -| subnet_self_links | Map of subnet self links keyed by 'region/name'. | | +| subnet_ips | Map of subnet address ranges keyed by name. | | +| subnet_regions | Map of subnet regions keyed by name. | | +| subnet_secondary_ranges | Map of subnet secondary ranges keyed by name. | | +| subnet_self_links | Map of subnet self links keyed by name. | | | subnets | Subnet resources. | | diff --git a/modules/net-vpc/main.tf b/modules/net-vpc/main.tf index 333518df..3c21f467 100644 --- a/modules/net-vpc/main.tf +++ b/modules/net-vpc/main.tf @@ -64,7 +64,7 @@ locals { } subnets = { for subnet in var.subnets : - "${subnet.region}/${subnet.name}" => subnet + "${subnet.region}/${subnet.name}" => subnet } } @@ -154,7 +154,7 @@ resource "google_compute_route" "gateway" { for_each = local.routes_gateway project = var.project_id network = google_compute_network.network.name - name = each.key + name = "${var.name}-${each.key}" description = "Terraform-managed." dest_range = each.value.dest_range priority = each.value.priority @@ -166,7 +166,7 @@ resource "google_compute_route" "ilb" { for_each = local.routes_ilb project = var.project_id network = google_compute_network.network.name - name = each.key + name = "${var.name}-${each.key}" description = "Terraform-managed." dest_range = each.value.dest_range priority = each.value.priority @@ -178,19 +178,21 @@ resource "google_compute_route" "instance" { for_each = local.routes_instance project = var.project_id network = google_compute_network.network.name - name = each.key + name = "${var.name}-${each.key}" description = "Terraform-managed." dest_range = each.value.dest_range priority = each.value.priority tags = each.value.tags next_hop_instance = each.value.next_hop + # not setting the instance zone will trigger a refresh + next_hop_instance_zone = regex("zones/([^/]+)/", each.value.next_hop)[0] } resource "google_compute_route" "ip" { for_each = local.routes_ip project = var.project_id network = google_compute_network.network.name - name = each.key + name = "${var.name}-${each.key}" description = "Terraform-managed." dest_range = each.value.dest_range priority = each.value.priority @@ -202,7 +204,7 @@ resource "google_compute_route" "vpn_tunnel" { for_each = local.routes_vpn_tunnel project = var.project_id network = google_compute_network.network.name - name = each.key + name = "${var.name}-${each.key}" description = "Terraform-managed." dest_range = each.value.dest_range priority = each.value.priority diff --git a/modules/net-vpc/variables.tf b/modules/net-vpc/variables.tf index 0632e8ac..d06eb401 100644 --- a/modules/net-vpc/variables.tf +++ b/modules/net-vpc/variables.tf @@ -35,19 +35,19 @@ variable "description" { variable "iam_roles" { description = "List of IAM roles keyed by subnet 'region/name'." type = map(list(string)) - default = null + default = {} } variable "iam_members" { description = "List of IAM members keyed by subnet 'region/name' and role." type = map(map(list(string))) - default = null + default = {} } variable "log_configs" { description = "Map keyed by subnet 'region/name' of optional configurations for flow logs when enabled." type = map(map(string)) - default = null + default = {} } variable "log_config_defaults" { @@ -93,7 +93,7 @@ variable "routes" { next_hop_type = string # gateway, instance, ip, vpn_tunnel, ilb next_hop = string })) - default = null + default = {} } variable "routing_mode" { diff --git a/modules/net-vpn-ha/main.tf b/modules/net-vpn-ha/main.tf index 751081ee..141bffc7 100644 --- a/modules/net-vpn-ha/main.tf +++ b/modules/net-vpn-ha/main.tf @@ -42,6 +42,7 @@ resource "google_compute_external_vpn_gateway" "external_gateway" { provider = google-beta count = var.peer_external_gateway != null ? 1 : 0 name = "external-${var.name}" + project = var.project_id redundancy_type = var.peer_external_gateway.redundancy_type description = "Terraform managed external VPN gateway" dynamic "interface" { diff --git a/modules/net-vpn-static/main.tf b/modules/net-vpn-static/main.tf index 559d8d95..91a5c23f 100644 --- a/modules/net-vpn-static/main.tf +++ b/modules/net-vpn-static/main.tf @@ -67,7 +67,7 @@ resource "google_compute_forwarding_rule" "udp-4500" { resource "google_compute_route" "route" { for_each = local.route_pairs - name = "vpn-${each.key}" + name = "vpn-${var.name}-${each.key}" project = var.project_id network = var.network dest_range = each.value.range diff --git a/modules/organization/README.md b/modules/organization/README.md index e38e62f9..bb22cdef 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -39,8 +39,7 @@ module "org" { | *access_policy_name* | Access Policy name. No Access Policy will be created. | string | | null | | *access_policy_title* | Access Policy title to be created. | string | | | | *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | -| *iam_additive_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map(list(string)) | | {} | -| *iam_additive_roles* | List of roles used to set non authoritative bindings. | list(string) | | [] | +| *iam_additive_bindings* | Map of roles lists used to set non authoritative bindings, keyed by members. | map(list(string)) | | {} | | *iam_audit_config* | Service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service. | map(map(list(string))) | | {} | | *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | | {} | | *iam_roles* | List of roles used to set authoritative bindings. | list(string) | | [] | diff --git a/modules/organization/main.tf b/modules/organization/main.tf index 83b29b8e..66add2bd 100644 --- a/modules/organization/main.tf +++ b/modules/organization/main.tf @@ -16,8 +16,8 @@ locals { iam_additive_pairs = flatten([ - for role in var.iam_additive_roles : [ - for member in lookup(var.iam_additive_members, role, []) : + for member, roles in var.iam_additive_bindings : [ + for role in roles : { role = role, member = member } ] ]) @@ -108,7 +108,7 @@ resource "google_organization_iam_binding" "authoritative" { } resource "google_organization_iam_member" "additive" { - for_each = length(var.iam_additive_roles) > 0 ? local.iam_additive : {} + for_each = length(var.iam_additive_bindings) > 0 ? local.iam_additive : {} org_id = var.org_id role = each.value.role member = each.value.member diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf index a9031384..4badecf8 100644 --- a/modules/organization/variables.tf +++ b/modules/organization/variables.tf @@ -44,18 +44,12 @@ variable "iam_roles" { default = [] } -variable "iam_additive_members" { - description = "Map of member lists used to set non authoritative bindings, keyed by role." +variable "iam_additive_bindings" { + description = "Map of roles lists used to set non authoritative bindings, keyed by members." type = map(list(string)) default = {} } -variable "iam_additive_roles" { - description = "List of roles used to set non authoritative bindings." - type = list(string) - default = [] -} - variable "iam_audit_config" { description = "Service audit logging configuration. Service as key, map of log permission (eg DATA_READ) and excluded members as value for each service." type = map(map(list(string))) diff --git a/modules/project/README.md b/modules/project/README.md index bdb8cfda..5d5c6bb3 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -24,6 +24,33 @@ module "project" { } ``` +### Minimal example with IAM additive roles + +```hcl +module "project" { + source = "./modules/project" + name = "project-example" + project_create = false + + iam_additive_bindings = { + "group:usergroup_watermlon_experimentation@lemonadeinc.io" = [ + "roles/viewer", + "roles/storage.objectAdmin" + ], + "group:usergroup_gcp_admin@lemonadeinc.io" = [ + "roles/owner", + ], + "group:usergroup_gcp_privilege_access@lemonadeinc.io" = [ + "roles/editor" + ], + "group:engineering@lemonadeinc.io" = [ + "roles/pubsub.subscriber", + "roles/storage.objectViewer" + ], + } +} +``` + ### Organization policies ```hcl @@ -58,12 +85,10 @@ module "project" { | name | description | type | required | default | |---|---|:---: |:---:|:---:| | name | Project name and id suffix. | string | ✓ | | -| parent | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | string | ✓ | | | *auto_create_network* | Whether to create the default network for the project | bool | | false | -| *billing_account* | Billing account id. | string | | | +| *billing_account* | Billing account id. | string | | null | | *custom_roles* | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | -| *iam_additive_members* | Map of member lists used to set non authoritative bindings, keyed by role. | map(list(string)) | | {} | -| *iam_additive_roles* | List of roles used to set non authoritative bindings. | list(string) | | [] | +| *iam_additive_bindings* | Map of roles lists used to set non authoritative bindings, keyed by members | map(list(string)) | | {} | | *iam_members* | Map of member lists used to set authoritative bindings, keyed by role. | map(list(string)) | | {} | | *iam_roles* | List of roles used to set authoritative bindings. | list(string) | | [] | | *labels* | Resource labels. | map(string) | | {} | @@ -71,9 +96,12 @@ module "project" { | *oslogin* | Enable OS Login. | bool | | false | | *oslogin_admins* | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string) | | [] | | *oslogin_users* | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string) | | [] | +| *parent* | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | | *policy_boolean* | Map of boolean org policies and enforcement value, set value to null for policy restore. | map(bool) | | {} | | *policy_list* | Map of list org policies, status is true for allow, false for deny, null for restore. Values can only be used for allow or deny. | map(object({...})) | | {} | | *prefix* | Prefix used to generate project id and name. | string | | null | +| *project_create* | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| *service_config* | Configure service API activation. | object({...}) | | ... | | *services* | Service APIs to enable. | list(string) | | [] | | *vpc_sc_perimeter* | Name of the VPC-SC perimeter the project belong to. | string | | null | | *vpc_sc_perimeter_bridges* | List of VPC-SC perimeter bridges the project belong to. Must be of the form accessPolicies/{policy_id}/servicePerimeters/{short_name} | list(string) | | [] | @@ -82,12 +110,10 @@ module "project" { | name | description | sensitive | |---|---|:---:| -| cloudsvc_service_account | Cloud services service account. | | | custom_roles | Ids of the created custom roles. | | -| gce_service_account | Default GCE service account. | | -| gcr_service_account | Default GCR service account. | | -| gke_service_account | Default GKE service account. | | -| name | Project ame. | | +| name | Project name. | | | number | Project number. | | | project_id | Project id. | | +| service_accounts | Product robot service accounts in project. | | + diff --git a/modules/project/main.tf b/modules/project/main.tf index 7f441bc6..b3bd29c8 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -15,13 +15,9 @@ */ locals { - cloudsvc_service_account = "${google_project.project.number}@cloudservices.gserviceaccount.com" - gce_service_account = "${google_project.project.number}-compute@developer.gserviceaccount.com" - gcr_service_account = "service-${google_project.project.number}@containerregistry.iam.gserviceaccount.com" - gke_service_account = "service-${google_project.project.number}@container-engine-robot.iam.gserviceaccount.com" iam_additive_pairs = flatten([ - for role in var.iam_additive_roles : [ - for member in lookup(var.iam_additive_members, role, []) : + for member, roles in var.iam_additive_bindings : [ + for role in roles : { role = role, member = member } ] ]) @@ -29,12 +25,23 @@ locals { for pair in local.iam_additive_pairs : "${pair.role}-${pair.member}" => pair } - parent_type = split("/", var.parent)[0] - parent_id = split("/", var.parent)[1] + parent_type = var.parent == null ? null : split("/", var.parent)[0] + parent_id = var.parent == null ? null : split("/", var.parent)[1] prefix = var.prefix == null ? "" : "${var.prefix}-" + project = ( + var.project_create + ? try(google_project.project.0, null) + : try(data.google_project.project.0, null) + ) +} + +data "google_project" "project" { + count = var.project_create ? 0 : 1 + project_id = "${local.prefix}${var.name}" } resource "google_project" "project" { + count = var.project_create ? 1 : 0 org_id = local.parent_type == "organizations" ? local.parent_id : null folder_id = local.parent_type == "folders" ? local.parent_id : null project_id = "${local.prefix}${var.name}" @@ -46,7 +53,7 @@ resource "google_project" "project" { resource "google_project_iam_custom_role" "roles" { for_each = var.custom_roles - project = google_project.project.project_id + project = local.project.project_id role_id = each.key title = "Custom role ${each.key}" description = "Terraform-managed" @@ -55,7 +62,7 @@ resource "google_project_iam_custom_role" "roles" { resource "google_compute_project_metadata_item" "oslogin_meta" { count = var.oslogin ? 1 : 0 - project = google_project.project.project_id + project = local.project.project_id key = "enable-oslogin" value = "TRUE" # depend on services or it will fail on destroy @@ -64,7 +71,7 @@ resource "google_compute_project_metadata_item" "oslogin_meta" { resource "google_resource_manager_lien" "lien" { count = var.lien_reason != "" ? 1 : 0 - parent = "projects/${google_project.project.number}" + parent = "projects/${local.project.number}" restrictions = ["resourcemanager.projects.delete"] origin = "created-by-terraform" reason = var.lien_reason @@ -72,10 +79,10 @@ resource "google_resource_manager_lien" "lien" { resource "google_project_service" "project_services" { for_each = toset(var.services) - project = google_project.project.project_id + project = local.project.project_id service = each.value - disable_on_destroy = true - disable_dependent_services = true + disable_on_destroy = var.service_config.disable_on_destroy + disable_dependent_services = var.service_config.disable_dependent_services } # IAM notes: @@ -85,7 +92,7 @@ resource "google_project_service" "project_services" { resource "google_project_iam_binding" "authoritative" { for_each = toset(var.iam_roles) - project = google_project.project.project_id + project = local.project.project_id role = each.value members = lookup(var.iam_members, each.value, []) depends_on = [ @@ -95,43 +102,47 @@ resource "google_project_iam_binding" "authoritative" { } resource "google_project_iam_member" "additive" { - for_each = length(var.iam_additive_roles) > 0 ? local.iam_additive : {} - project = google_project.project.project_id + for_each = length(var.iam_additive_bindings) > 0 ? local.iam_additive : {} + project = local.project.project_id role = each.value.role member = each.value.member + depends_on = [ + google_project_service.project_services, + google_project_iam_custom_role.roles + ] } resource "google_project_iam_member" "oslogin_iam_serviceaccountuser" { for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) - project = google_project.project.project_id + project = local.project.project_id role = "roles/iam.serviceAccountUser" member = each.value } resource "google_project_iam_member" "oslogin_compute_viewer" { for_each = var.oslogin ? toset(distinct(concat(var.oslogin_admins, var.oslogin_users))) : toset([]) - project = google_project.project.project_id + project = local.project.project_id role = "roles/compute.viewer" member = each.value } resource "google_project_iam_member" "oslogin_admins" { for_each = var.oslogin ? toset(var.oslogin_admins) : toset([]) - project = google_project.project.project_id + project = local.project.project_id role = "roles/compute.osAdminLogin" member = each.value } resource "google_project_iam_member" "oslogin_users" { for_each = var.oslogin ? toset(var.oslogin_users) : toset([]) - project = google_project.project.project_id + project = local.project.project_id role = "roles/compute.osLogin" member = each.value } resource "google_project_organization_policy" "boolean" { for_each = var.policy_boolean - project = google_project.project.project_id + project = local.project.project_id constraint = each.key dynamic boolean_policy { @@ -152,7 +163,7 @@ resource "google_project_organization_policy" "boolean" { resource "google_project_organization_policy" "list" { for_each = var.policy_list - project = google_project.project.project_id + project = local.project.project_id constraint = each.key dynamic list_policy { diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf index c0751677..2edeecaf 100644 --- a/modules/project/outputs.tf +++ b/modules/project/outputs.tf @@ -16,7 +16,7 @@ output "project_id" { description = "Project id." - value = google_project.project.project_id + value = try(local.project.project_id, null) depends_on = [ google_project_organization_policy.boolean, google_project_organization_policy.list, @@ -25,8 +25,8 @@ output "project_id" { } output "name" { - description = "Project ame." - value = google_project.project.name + description = "Project name." + value = local.project.name depends_on = [ google_project_organization_policy.boolean, google_project_organization_policy.list, @@ -36,7 +36,7 @@ output "name" { output "number" { description = "Project number." - value = google_project.project.number + value = local.project.number depends_on = [ google_project_organization_policy.boolean, google_project_organization_policy.list, @@ -44,31 +44,20 @@ output "number" { ] } -output "cloudsvc_service_account" { - description = "Cloud services service account." - value = "${local.cloudsvc_service_account}" - depends_on = [google_project_service.project_services] -} - -output "gce_service_account" { - description = "Default GCE service account." - value = local.gce_service_account - depends_on = [google_project_service.project_services] -} - -output "gcr_service_account" { - description = "Default GCR service account." - value = local.gcr_service_account - depends_on = [google_project_service.project_services] -} - -output "gke_service_account" { - description = "Default GKE service account." - value = local.gke_service_account - depends_on = [google_project_service.project_services] +output "service_accounts" { + description = "Product robot service accounts in project." + value = { + cloud_services = local.service_account_cloud_services + default = local.service_accounts_default + robots = local.service_accounts_robots + } + depends_on = [google_project_service.project_services] } output "custom_roles" { description = "Ids of the created custom roles." - value = [for role in google_project_iam_custom_role.roles : role.role_id] + value = { + for name, role in google_project_iam_custom_role.roles : + name => role.id + } } diff --git a/modules/project/service_accounts.tf b/modules/project/service_accounts.tf new file mode 100644 index 00000000..7e0f97c1 --- /dev/null +++ b/modules/project/service_accounts.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + service_account_cloud_services = "${local.project.number}@cloudservices.gserviceaccount.com" + service_accounts_default = { + # TODO: Find a better place to store BQ service account + bq = "bq-${local.project.number}@bigquery-encryption.iam.gserviceaccount.com" + compute = "${local.project.number}-compute@developer.gserviceaccount.com" + gae = "${local.project.project_id}@appspot.gserviceaccount.com" + } + service_accounts_robot_services = { + cloudasset = "gcp-sa-cloudasset" + cloudbuild = "gcp-sa-cloudbuild" + compute = "compute-system" + container-engine = "container-engine-robot" + containerregistry = "containerregistry" + dataflow = "dataflow-service-producer-prod" + dataproc = "dataproc-accounts" + gae-flex = "gae-api-prod" + gcf = "gcf-admin-robot" + pubsub = "gcp-sa-pubsub" + storage = "gs-project-accounts" + } + service_accounts_robots = { + for service, name in local.service_accounts_robot_services : + service => "service-${local.project.number}@${name}.iam.gserviceaccount.com" + } +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 769d3ff0..e7507037 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -23,7 +23,7 @@ variable "auto_create_network" { variable "billing_account" { description = "Billing account id." type = string - default = "" + default = null } variable "custom_roles" { @@ -44,18 +44,13 @@ variable "iam_roles" { default = [] } -variable "iam_additive_members" { - description = "Map of member lists used to set non authoritative bindings, keyed by role." + +variable "iam_additive_bindings" { + description = "Map of roles lists used to set non authoritative bindings, keyed by members" type = map(list(string)) default = {} } -variable "iam_additive_roles" { - description = "List of roles used to set non authoritative bindings." - type = list(string) - default = [] -} - variable "labels" { description = "Resource labels." type = map(string) @@ -92,8 +87,9 @@ variable "oslogin_users" { } variable "parent" { - description = "The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id." + description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." type = string + default = null } variable "policy_boolean" { @@ -119,12 +115,30 @@ variable "prefix" { default = null } +variable "project_create" { + description = "Create project. When set to false, uses a data source to reference existing project." + type = bool + default = true +} + variable "services" { description = "Service APIs to enable." type = list(string) default = [] } +variable "service_config" { + description = "Configure service API activation." + type = object({ + disable_on_destroy = bool + disable_dependent_services = bool + }) + default = { + disable_on_destroy = true + disable_dependent_services = true + } +} + variable "vpc_sc_perimeter" { description = < +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | PubSub topic name. | string | ✓ | | +| project_id | Project used for resources. | string | ✓ | | +| *dead_letter_configs* | Per-subscription dead letter policy configuration. | map(object({...})) | | {} | +| *defaults* | Subscription defaults for options. | object({...}) | | ... | +| *iam_members* | IAM members for each topic role. | map(list(string)) | | {} | +| *iam_roles* | IAM roles for topic. | list(string) | | [] | +| *kms_key* | KMS customer managed encryption key. | string | | null | +| *labels* | Labels. | map(string) | | {} | +| *push_configs* | Push subscription configurations. | map(object({...})) | | {} | +| *regions* | List of regions used to set persistence policy. | list(string) | | [] | +| *subscription_iam_members* | IAM members for each subscription and role. | map(map(list(string))) | | {} | +| *subscription_iam_roles* | IAM roles for each subscription. | map(list(string)) | | {} | +| *subscriptions* | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({...})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | Topic id. | | +| subscription_id | Subscription ids. | | +| subscriptions | Subscription resources. | | +| topic | Topic resource. | | + diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf new file mode 100644 index 00000000..50876070 --- /dev/null +++ b/modules/pubsub/main.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + iam_pairs = var.subscription_iam_roles == null ? [] : flatten([ + for name, roles in var.subscription_iam_roles : + [for role in roles : { name = name, role = role }] + ]) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.name}-${pair.role}" => pair + } + iam_members = ( + var.subscription_iam_members == null ? {} : var.subscription_iam_members + ) + oidc_config = { + for k, v in var.push_configs : k => v.oidc_token + } + subscriptions = { + for k, v in var.subscriptions : k => { + labels = try(v.labels, v, null) == null ? var.labels : v.labels + options = try(v.options, v, null) == null ? var.defaults : v.options + } + } +} + +resource "google_pubsub_topic" "default" { + project = var.project_id + name = var.name + kms_key_name = var.kms_key + labels = var.labels + + dynamic message_storage_policy { + for_each = length(var.regions) > 0 ? [var.regions] : [] + content { + allowed_persistence_regions = var.regions + } + } +} + +resource "google_pubsub_topic_iam_binding" "default" { + for_each = toset(var.iam_roles) + project = var.project_id + topic = google_pubsub_topic.default.name + role = each.value + members = lookup(var.iam_members, each.value, []) +} + +resource "google_pubsub_subscription" "default" { + for_each = local.subscriptions + project = var.project_id + name = each.key + topic = google_pubsub_topic.default.name + labels = each.value.labels + ack_deadline_seconds = each.value.options.ack_deadline_seconds + message_retention_duration = each.value.options.message_retention_duration + retain_acked_messages = each.value.options.retain_acked_messages + + dynamic expiration_policy { + for_each = each.value.options.expiration_policy_ttl == null ? [] : [""] + content { + ttl = each.value.options.expiration_policy_ttl + } + } + + dynamic dead_letter_policy { + for_each = try(var.dead_letter_configs[each.key], null) == null ? [] : [""] + content { + dead_letter_topic = var.dead_letter_configs[each.key].topic + max_delivery_attempts = var.dead_letter_configs[each.key].max_delivery_attempts + } + } + + dynamic push_config { + for_each = try(var.push_configs[each.key], null) == null ? [] : [""] + content { + push_endpoint = var.push_configs[each.key].endpoint + attributes = var.push_configs[each.key].attributes + dynamic oidc_token { + for_each = ( + local.oidc_config[each.key] == null ? [] : [""] + ) + content { + service_account_email = local.oidc_config[each.key].service_account_email + audience = local.oidc_config[each.key].audience + } + } + } + } +} + +resource "google_pubsub_subscription_iam_binding" "default" { + for_each = local.iam_keypairs + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.name].name + role = each.value.role + members = lookup( + lookup(local.iam_members, each.value.name, {}), each.value.role, [] + ) +} diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf new file mode 100644 index 00000000..950f60b2 --- /dev/null +++ b/modules/pubsub/outputs.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Topic id." + value = google_pubsub_topic.default.id +} + +output "subscriptions" { + description = "Subscription resources." + value = google_pubsub_subscription.default + depends_on = [ + google_pubsub_subscription_iam_binding.default + ] +} + +output "subscription_id" { + description = "Subscription ids." + value = { + for k, v in google_pubsub_subscription.default : k => v.id + } + depends_on = [ + google_pubsub_subscription_iam_binding.default + ] +} + +output "topic" { + description = "Topic resource." + value = google_pubsub_topic.default + depends_on = [ + google_pubsub_topic_iam_binding.default + ] +} diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf new file mode 100644 index 00000000..d642c7a6 --- /dev/null +++ b/modules/pubsub/variables.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "dead_letter_configs" { + description = "Per-subscription dead letter policy configuration." + type = map(object({ + topic = string + max_delivery_attemps = number + })) + default = {} +} + +variable "defaults" { + description = "Subscription defaults for options." + type = object({ + ack_deadline_seconds = number + message_retention_duration = number + retain_acked_messages = bool + expiration_policy_ttl = string + }) + default = { + ack_deadline_seconds = null + message_retention_duration = null + retain_acked_messages = null + expiration_policy_ttl = null + } +} + +variable "iam_members" { + description = "IAM members for each topic role." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "IAM roles for topic." + type = list(string) + default = [] +} + +variable "kms_key" { + description = "KMS customer managed encryption key." + type = string + default = null +} + +variable "labels" { + description = "Labels." + type = map(string) + default = {} +} + +variable "name" { + description = "PubSub topic name." + type = string +} + +variable "regions" { + description = "List of regions used to set persistence policy." + type = list(string) + default = [] +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + + +variable "push_configs" { + description = "Push subscription configurations." + type = map(object({ + attributes = map(string) + endpoint = string + oidc_token = object({ + audience = string + service_account_email = string + }) + })) + default = {} +} + +variable "subscriptions" { + description = "Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null." + type = map(object({ + labels = map(string) + options = object({ + ack_deadline_seconds = number + message_retention_duration = number + retain_acked_messages = bool + expiration_policy_ttl = string + }) + })) + default = {} +} + +variable "subscription_iam_members" { + description = "IAM members for each subscription and role." + type = map(map(list(string))) + default = {} +} + +variable "subscription_iam_roles" { + description = "IAM roles for each subscription." + type = map(list(string)) + default = {} +} diff --git a/modules/pubsub/versions.tf b/modules/pubsub/versions.tf new file mode 100644 index 00000000..ce6918e0 --- /dev/null +++ b/modules/pubsub/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/secret-manager/README.md b/modules/secret-manager/README.md index 413fedba..b8224c53 100644 --- a/modules/secret-manager/README.md +++ b/modules/secret-manager/README.md @@ -84,7 +84,7 @@ module "secret-manager" { | *iam_roles* | IAM roles keyed by secret name. | map(list(string)) | | {} | | *labels* | Optional labels for each secret. | map(map(string)) | | {} | | *secrets* | Map of secrets to manage and their locations. If locations is null, automatic management will be set. | map(list(string)) | | {} | -| *versions* | Optional versions to manage for each secret. Version names are only used internally to track each version and must be unique for each secret/version pair. | map(list(object({...}))) | | {} | +| *versions* | Optional versions to manage for each secret. Version names are only used internally to track individual versions. | map(map(object({...}))) | | {} | ## Outputs diff --git a/modules/service-directory/README.md b/modules/service-directory/README.md new file mode 100644 index 00000000..c57bd9f8 --- /dev/null +++ b/modules/service-directory/README.md @@ -0,0 +1,125 @@ +# Google Cloud Service Directory Module + +This module allows managing a single [Service Directory](https://cloud.google.com/service-directory) namespace, including multiple services, endpoints and IAM bindings at the namespace and service levels. + +It can be used in conjunction with the [DNS](../dns) module to create [service-directory based DNS zones](https://cloud.google.com/service-directory/docs/configuring-service-directory-zone, offloading IAM control of `A` and `SRV` records at the namespace or service level to Service Directory. The last examples shows how to wire the two modules together. + + +## Examples + +### Namespace with IAM + +```hcl +module "service-directory" { + source = "./modules/service-directory" + project_id = "my-project" + location = "europe-west1" + name = "sd-1" + iam_members = { + "roles/servicedirectory.editor" = [ + "serviceAccount:namespace-editor@example.com" + ] + } + iam_roles = [ + "roles/servicedirectory.editor" + ] +} +``` + +### Services with IAM and endpoints + +```hcl +module "service-directory" { + source = "./modules/service-directory" + project_id = "my-project" + location = "europe-west1" + name = "sd-1" + services = { + one = { + endpoints = ["first", "second"] + metadata = null + } + } + service_iam_members = { + one = { + "roles/servicedirectory.editor" = [ + "serviceAccount:service-editor.example.com" + ] + } + } + service_iam_roles = { + one = ["roles/servicedirectory.editor"] + } + endpoint_config = { + "one/first" = { address = "127.0.0.1", port = 80, metadata = {} } + "one/second" = { address = "127.0.0.2", port = 80, metadata = {} } + } +} +``` + +### DNS based zone + +Wiring a service directory namespace to a private DNS zone allows querying the namespace, and delegating control of DNS records at the namespace or service level. This effectively allows fine grained ACL control of Cloud DNS zones. + +```hcl +module "service-directory" { + source = "./modules/service-directory" + project_id = "my-project" + location = "europe-west1" + name = "apps" + iam_members = { + "roles/servicedirectory.editor" = [ + "serviceAccount:namespace-editor@example.com" + ] + } + iam_roles = [ + "roles/servicedirectory.editor" + ] + services = { + app1 = { endpoints = ["one"], metadata = null } + } + endpoint_config = { + "app1/one" = { address = "127.0.0.1", port = 80, metadata = {} } + } +} + +module "dns-sd" { + source = "./modules/dns" + project_id = "my-project" + type = "service-directory" + name = "apps" + domain = "apps.example.org." + client_networks = [local.vpc_self_link] + service_directory_namespace = module.service-directory.id +} + +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| location | Namespace location. | string | ✓ | | +| name | Namespace name. | string | ✓ | | +| project_id | Project used for resources. | string | ✓ | | +| *endpoint_config* | Map of endpoint attributes, keys are in service/endpoint format. | map(object({...})) | | {} | +| *iam_members* | IAM members for each namespace role. | map(list(string)) | | {} | +| *iam_roles* | IAM roles for the namespace. | list(string) | | [] | +| *labels* | Labels. | map(string) | | {} | +| *service_iam_members* | IAM members for each service and role. | map(map(list(string))) | | {} | +| *service_iam_roles* | IAM roles for each service. | map(list(string)) | | {} | +| *services* | Service configuration, using service names as keys. | map(object({...})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| endpoints | Endpoint resources. | | +| id | Namespace id (short name). | | +| name | Namespace name (long name). | | +| namespace | Namespace resource. | | +| service_id | Service ids (short names). | | +| service_names | Service ids (long names). | | +| services | Service resources. | | + diff --git a/modules/service-directory/main.tf b/modules/service-directory/main.tf new file mode 100644 index 00000000..8348de44 --- /dev/null +++ b/modules/service-directory/main.tf @@ -0,0 +1,81 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + endpoint_list = flatten([ + for name, attrs in var.services : [ + for endpoint in attrs.endpoints : { service : name, endpoint : endpoint } + ] + ]) + endpoints = { + for ep in local.endpoint_list : "${ep.service}/${ep.endpoint}" => ep + } + iam_pairs = var.service_iam_roles == null ? [] : flatten([ + for name, roles in var.service_iam_roles : + [for role in roles : { name = name, role = role }] + ]) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.name}-${pair.role}" => pair + } + iam_members = ( + var.service_iam_members == null ? {} : var.service_iam_members + ) +} + +resource "google_service_directory_namespace" "default" { + provider = google-beta + project = var.project_id + namespace_id = var.name + location = var.location + labels = var.labels +} + +resource "google_service_directory_namespace_iam_binding" "default" { + provider = google-beta + for_each = toset(var.iam_roles) + name = google_service_directory_namespace.default.name + role = each.value + members = lookup(var.iam_members, each.value, []) +} + +resource "google_service_directory_service" "default" { + provider = google-beta + for_each = var.services + namespace = google_service_directory_namespace.default.id + service_id = each.key + metadata = each.value.metadata +} + +resource "google_service_directory_service_iam_binding" "default" { + provider = google-beta + for_each = local.iam_keypairs + name = google_service_directory_service.default[each.value.name].name + role = each.value.role + members = lookup( + lookup(local.iam_members, each.value.name, {}), each.value.role, [] + ) +} + +resource "google_service_directory_endpoint" "default" { + provider = google-beta + for_each = local.endpoints + endpoint_id = each.value.endpoint + service = google_service_directory_service.default[each.value.service].id + metadata = try(var.endpoint_config[each.key].metadata, null) + address = try(var.endpoint_config[each.key].address, null) + port = try(var.endpoint_config[each.key].port, null) +} diff --git a/modules/service-directory/outputs.tf b/modules/service-directory/outputs.tf new file mode 100644 index 00000000..0fab3996 --- /dev/null +++ b/modules/service-directory/outputs.tf @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "endpoints" { + description = "Endpoint resources." + value = google_service_directory_endpoint.default +} + +output "id" { + description = "Namespace id (short name)." + value = google_service_directory_namespace.default.id +} + +output "name" { + description = "Namespace name (long name)." + value = google_service_directory_namespace.default.name +} + +output "namespace" { + description = "Namespace resource." + value = google_service_directory_namespace.default + depends_on = [ + google_service_directory_namespace_iam_binding.default + ] +} + +output "services" { + description = "Service resources." + value = google_service_directory_service.default + depends_on = [ + google_service_directory_service_iam_binding.default + ] +} + +output "service_id" { + description = "Service ids (short names)." + value = { + for k, v in google_service_directory_service.default : k => v.id + } + depends_on = [ + google_service_directory_service_iam_binding.default + ] +} + +output "service_names" { + description = "Service ids (long names)." + value = { + for k, v in google_service_directory_service.default : k => v.name + } + depends_on = [ + google_service_directory_service_iam_binding.default + ] +} diff --git a/modules/service-directory/variables.tf b/modules/service-directory/variables.tf new file mode 100644 index 00000000..8b921d56 --- /dev/null +++ b/modules/service-directory/variables.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# we need a separate variable as address will be dynamic in most cases +variable "endpoint_config" { + description = "Map of endpoint attributes, keys are in service/endpoint format." + type = map(object({ + address = string + port = number + metadata = map(string) + })) + default = {} +} + +variable "iam_members" { + description = "IAM members for each namespace role." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "IAM roles for the namespace." + type = list(string) + default = [] +} + +variable "labels" { + description = "Labels." + type = map(string) + default = {} +} + +variable "location" { + description = "Namespace location." + type = string +} + +variable "name" { + description = "Namespace name." + type = string +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "service_iam_members" { + description = "IAM members for each service and role." + type = map(map(list(string))) + default = {} +} + +variable "service_iam_roles" { + description = "IAM roles for each service." + type = map(list(string)) + default = {} +} + +variable "services" { + description = "Service configuration, using service names as keys." + type = map(object({ + endpoints = list(string) + metadata = map(string) + })) + default = {} +} diff --git a/modules/service-directory/versions.tf b/modules/service-directory/versions.tf new file mode 100644 index 00000000..ce6918e0 --- /dev/null +++ b/modules/service-directory/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/modules/source-repository/README.md b/modules/source-repository/README.md new file mode 100644 index 00000000..c5a2eb85 --- /dev/null +++ b/modules/source-repository/README.md @@ -0,0 +1,38 @@ +# Google Cloud Source Repository Module + +This module allows managing a single Cloud Source Repository, including IAM bindings. + + +## Examples + +### Simple repository with IAM + +```hcl +module "repo" { + source e = "./modules/source-repository" + project_id = "my-project" + name = "my-repo" + iam_roles = ["roles/source.reader"] + iam_members = { + "roles/source.reader" = ["user:foo@example.com"] + } +} +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---: |:---:|:---:| +| name | Repository topic name. | string | ✓ | | +| project_id | Project used for resources. | string | ✓ | | +| *iam_members* | IAM members for each topic role. | map(list(string)) | | {} | +| *iam_roles* | IAM roles for topic. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| id | Repository id. | | +| url | Repository URL. | | + diff --git a/modules/source-repository/main.tf b/modules/source-repository/main.tf new file mode 100644 index 00000000..810b4482 --- /dev/null +++ b/modules/source-repository/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_sourcerepo_repository" "default" { + project = var.project_id + name = var.name +} + +resource "google_sourcerepo_repository_iam_binding" "default" { + for_each = toset(var.iam_roles) + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.value + members = lookup(var.iam_members, each.value, []) + + depends_on = [ + google_sourcerepo_repository.default + ] +} diff --git a/modules/source-repository/outputs.tf b/modules/source-repository/outputs.tf new file mode 100644 index 00000000..023c2672 --- /dev/null +++ b/modules/source-repository/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Repository id." + value = google_sourcerepo_repository.default.id +} + +output "url" { + description = "Repository URL." + value = google_sourcerepo_repository.default.url +} diff --git a/modules/source-repository/variables.tf b/modules/source-repository/variables.tf new file mode 100644 index 00000000..1725d4b4 --- /dev/null +++ b/modules/source-repository/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "iam_members" { + description = "IAM members for each topic role." + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + description = "IAM roles for topic." + type = list(string) + default = [] +} + +variable "name" { + description = "Repository topic name." + type = string +} diff --git a/modules/source-repository/versions.tf b/modules/source-repository/versions.tf new file mode 100644 index 00000000..ce6918e0 --- /dev/null +++ b/modules/source-repository/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12.6" +} diff --git a/tests/modules/compute_mig/test_plan.py b/tests/modules/compute_mig/test_plan.py index c581750e..4d95eda8 100644 --- a/tests/modules/compute_mig/test_plan.py +++ b/tests/modules/compute_mig/test_plan.py @@ -21,53 +21,55 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') def test_defaults(plan_runner): - "Test variable defaults." - _, resources = plan_runner(FIXTURES_DIR) - assert len(resources) == 1 - mig = resources[0] - assert mig['type'] == 'google_compute_instance_group_manager' - assert mig['values']['target_size'] == 2 - assert mig['values']['zone'] - _, resources = plan_runner(FIXTURES_DIR, regional='true') - assert len(resources) == 1 - mig = resources[0] - assert mig['type'] == 'google_compute_region_instance_group_manager' - assert mig['values']['target_size'] == 2 - assert mig['values']['region'] + "Test variable defaults." + _, resources = plan_runner(FIXTURES_DIR) + assert len(resources) == 1 + mig = resources[0] + assert mig['type'] == 'google_compute_instance_group_manager' + assert mig['values']['target_size'] == 2 + assert mig['values']['zone'] + _, resources = plan_runner(FIXTURES_DIR, regional='true') + assert len(resources) == 1 + mig = resources[0] + assert mig['type'] == 'google_compute_region_instance_group_manager' + assert mig['values']['target_size'] == 2 + assert mig['values']['region'] def test_health_check(plan_runner): - "Test health check resource." - health_check_config = '{type="tcp", check={port=80}, config=null, logging=false}' - _, resources = plan_runner( - FIXTURES_DIR, health_check_config=health_check_config) - assert len(resources) == 2 - assert any(r['type'] == 'google_compute_health_check' for r in resources) + "Test health check resource." + health_check_config = '{type="tcp", check={port=80}, config=null, logging=false}' + _, resources = plan_runner( + FIXTURES_DIR, health_check_config=health_check_config) + assert len(resources) == 2 + assert any(r['type'] == 'google_compute_health_check' for r in resources) def test_autoscaler(plan_runner): - "Test autoscaler resource." - autoscaler_config = ( - '{' - 'max_replicas=3, min_replicas=1, cooldown_period=60,' - 'cpu_utilization_target=65, load_balancing_utilization_target=null,' - 'metric=null' - '}' - ) - _, resources = plan_runner(FIXTURES_DIR, autoscaler_config=autoscaler_config) - assert len(resources) == 2 - autoscaler = resources[0] - assert autoscaler['type'] == 'google_compute_autoscaler' - assert autoscaler['values']['autoscaling_policy'] == [{ - 'cooldown_period': 60, - 'cpu_utilization': [{'target': 65}], - 'load_balancing_utilization': [], - 'max_replicas': 3, - 'metric': [], - 'min_replicas': 1 - }] - _, resources = plan_runner( - FIXTURES_DIR, autoscaler_config=autoscaler_config, regional='true') - assert len(resources) == 2 - autoscaler = resources[0] - assert autoscaler['type'] == 'google_compute_region_autoscaler' + "Test autoscaler resource." + autoscaler_config = ( + '{' + 'max_replicas=3, min_replicas=1, cooldown_period=60,' + 'cpu_utilization_target=65, load_balancing_utilization_target=null,' + 'metric=null' + '}' + ) + _, resources = plan_runner( + FIXTURES_DIR, autoscaler_config=autoscaler_config) + assert len(resources) == 2 + autoscaler = resources[0] + assert autoscaler['type'] == 'google_compute_autoscaler' + assert autoscaler['values']['autoscaling_policy'] == [{ + 'cooldown_period': 60, + 'cpu_utilization': [{'target': 65}], + 'load_balancing_utilization': [], + 'max_replicas': 3, + 'metric': [], + 'min_replicas': 1, + 'mode': 'ON' + }] + _, resources = plan_runner( + FIXTURES_DIR, autoscaler_config=autoscaler_config, regional='true') + assert len(resources) == 2 + autoscaler = resources[0] + assert autoscaler['type'] == 'google_compute_region_autoscaler' diff --git a/tests/modules/compute_vm/fixture/main.tf b/tests/modules/compute_vm/fixture/main.tf index a1348834..41999d08 100644 --- a/tests/modules/compute_vm/fixture/main.tf +++ b/tests/modules/compute_vm/fixture/main.tf @@ -30,4 +30,6 @@ module "test" { instance_count = var.instance_count use_instance_template = var.use_instance_template group = var.group + iam_roles = var.iam_roles + iam_members = var.iam_members } diff --git a/tests/modules/compute_vm/fixture/variables.tf b/tests/modules/compute_vm/fixture/variables.tf index 7191d315..118383fb 100644 --- a/tests/modules/compute_vm/fixture/variables.tf +++ b/tests/modules/compute_vm/fixture/variables.tf @@ -19,6 +19,16 @@ variable "group" { default = null } +variable "iam_members" { + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + type = list(string) + default = [] +} + variable "instance_count" { type = number default = 1 diff --git a/tests/modules/compute_vm/test_plan.py b/tests/modules/compute_vm/test_plan.py index 733ee1ee..a37b55ac 100644 --- a/tests/modules/compute_vm/test_plan.py +++ b/tests/modules/compute_vm/test_plan.py @@ -21,35 +21,58 @@ FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') def test_single_instance(plan_runner): - plan, resources = plan_runner(FIXTURES_DIR) + _, resources = plan_runner(FIXTURES_DIR) assert len(resources) == 1 assert resources[0]['type'] == 'google_compute_instance' def test_multiple_instances(plan_runner): - plan, resources = plan_runner(FIXTURES_DIR, instance_count=2) + _, resources = plan_runner(FIXTURES_DIR, instance_count=2) assert len(resources) == 2 assert set(r['type'] for r in resources) == set(['google_compute_instance']) def test_service_account(plan_runner): - plan, resources = plan_runner(FIXTURES_DIR, instance_count=2, - service_account_create='true') + _, resources = plan_runner(FIXTURES_DIR, instance_count=2, + service_account_create='true') assert len(resources) == 3 assert 'google_service_account' in [r['type'] for r in resources] def test_template(plan_runner): - plan, resources = plan_runner(FIXTURES_DIR, use_instance_template='true') + _, resources = plan_runner(FIXTURES_DIR, use_instance_template='true') assert len(resources) == 1 assert resources[0]['type'] == 'google_compute_instance_template' assert resources[0]['values']['name_prefix'] == 'test-' def test_group(plan_runner): - plan, resources = plan_runner(FIXTURES_DIR, instance_count=2, - group='{named_ports={}}') + _, resources = plan_runner(FIXTURES_DIR, instance_count=2, + group='{named_ports={}}') assert len(resources) == 3 assert set(r['type'] for r in resources) == set([ 'google_compute_instance_group', 'google_compute_instance' ]) + + +def test_iam(plan_runner): + iam_roles = '["roles/compute.instanceAdmin", "roles/iam.serviceAccountUser"]' + iam_members = ( + '{"roles/compute.instanceAdmin" = ["user:a@a.com", "user:b@a.com"],' + '"roles/iam.serviceAccountUser" = ["user:a@a.com"]}' + ) + _, resources = plan_runner( + FIXTURES_DIR, instance_count=2, iam_roles=iam_roles, iam_members=iam_members) + assert len(resources) == 6 + assert set(r['type'] for r in resources) == set([ + 'google_compute_instance', 'google_compute_instance_iam_binding']) + iam_bindings = dict( + (r['index'], r['values']['members']) for r in resources if r['type'] + == 'google_compute_instance_iam_binding' + ) + assert iam_bindings == { + 'roles/compute.instanceAdmin/test-1': ['user:a@a.com', 'user:b@a.com'], + 'roles/compute.instanceAdmin/test-2': ['user:a@a.com', 'user:b@a.com'], + 'roles/iam.serviceAccountUser/test-1': ['user:a@a.com'], + 'roles/iam.serviceAccountUser/test-2': ['user:a@a.com'], + } diff --git a/tests/modules/gcs/fixture/variables.tf b/tests/modules/gcs/fixture/variables.tf index 7caec8c7..08e95396 100644 --- a/tests/modules/gcs/fixture/variables.tf +++ b/tests/modules/gcs/fixture/variables.tf @@ -41,7 +41,7 @@ variable "labels" { variable "prefix" { type = string - default = "" + default = null } variable "storage_class" { diff --git a/tests/modules/iam_service_accounts/__init__.py b/tests/modules/iam_service_accounts/__init__.py new file mode 100644 index 00000000..6913f02e --- /dev/null +++ b/tests/modules/iam_service_accounts/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/modules/iam_service_accounts/fixture/main.tf b/tests/modules/iam_service_accounts/fixture/main.tf new file mode 100644 index 00000000..69188086 --- /dev/null +++ b/tests/modules/iam_service_accounts/fixture/main.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../modules/iam-service-accounts" + project_id = var.project_id + names = ["sa-one", "sa-two", "sa-three"] + prefix = var.prefix + generate_keys = var.generate_keys + iam_members = var.iam_members + iam_roles = var.iam_roles + iam_billing_roles = var.iam_billing_roles + iam_folder_roles = var.iam_folder_roles + iam_organization_roles = var.iam_organization_roles + iam_project_roles = var.iam_project_roles + iam_storage_roles = var.iam_storage_roles +} diff --git a/tests/modules/iam_service_accounts/fixture/variables.tf b/tests/modules/iam_service_accounts/fixture/variables.tf new file mode 100644 index 00000000..0a784a94 --- /dev/null +++ b/tests/modules/iam_service_accounts/fixture/variables.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "generate_keys" { + type = bool + default = false +} + +variable "iam_members" { + type = map(list(string)) + default = {} +} + +variable "iam_roles" { + type = list(string) + default = [] +} + +variable "iam_billing_roles" { + type = map(list(string)) + default = {} +} + +variable "iam_folder_roles" { + type = map(list(string)) + default = {} +} + +variable "iam_organization_roles" { + type = map(list(string)) + default = {} +} + +variable "iam_project_roles" { + type = map(list(string)) + default = {} +} + +variable "iam_storage_roles" { + type = map(list(string)) + default = {} +} + +variable "prefix" { + type = string + default = null +} + +variable "project_id" { + type = string + default = "my-project" +} diff --git a/tests/modules/iam_service_accounts/test_plan.py b/tests/modules/iam_service_accounts/test_plan.py new file mode 100644 index 00000000..dbcb2bef --- /dev/null +++ b/tests/modules/iam_service_accounts/test_plan.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +def test_resources(plan_runner): + "Test service account resource." + _, resources = plan_runner(FIXTURES_DIR) + assert len(resources) == 3 + assert set(r['type'] for r in resources) == set(['google_service_account']) + assert set(r['values']['account_id'] for r in resources) == set([ + 'sa-one', 'sa-two', 'sa-three' + ]) + _, resources = plan_runner(FIXTURES_DIR, prefix='foo') + assert set(r['values']['account_id'] for r in resources) == set([ + 'foo-sa-one', 'foo-sa-two', 'foo-sa-three' + ]) + + +def test_iam_roles(plan_runner): + "Test iam roles with no memmbers." + _, resources = plan_runner(FIXTURES_DIR, + iam_roles='["roles/iam.serviceAccountUser"]') + assert len(resources) == 6 + iam_resources = [r for r in resources if r['type'] + != 'google_service_account'] + assert len(iam_resources) == 3 + assert set(r['type'] for r in iam_resources) == set( + ['google_service_account_iam_binding']) + assert [r['index'] for r in iam_resources] == [ + 'sa-one-roles/iam.serviceAccountUser', + 'sa-three-roles/iam.serviceAccountUser', + 'sa-two-roles/iam.serviceAccountUser', + ] diff --git a/tests/modules/kms/__init__.py b/tests/modules/kms/__init__.py new file mode 100644 index 00000000..6913f02e --- /dev/null +++ b/tests/modules/kms/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/modules/kms/fixture/main.tf b/tests/modules/kms/fixture/main.tf new file mode 100644 index 00000000..f027f978 --- /dev/null +++ b/tests/modules/kms/fixture/main.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../modules/kms" + iam_members = var.iam_members + iam_roles = var.iam_roles + key_iam_members = var.key_iam_members + key_iam_roles = var.key_iam_roles + key_purpose = var.key_purpose + key_purpose_defaults = var.key_purpose_defaults + keyring = var.keyring + keyring_create = var.keyring_create + keys = var.keys + project_id = var.project_id +} diff --git a/tests/modules/kms/fixture/outputs.tf b/tests/modules/kms/fixture/outputs.tf new file mode 100644 index 00000000..77b8211f --- /dev/null +++ b/tests/modules/kms/fixture/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "module" { + value = module.test +} diff --git a/tests/modules/kms/fixture/variables.tf b/tests/modules/kms/fixture/variables.tf new file mode 100644 index 00000000..10f3f318 --- /dev/null +++ b/tests/modules/kms/fixture/variables.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam_members" { + type = map(list(string)) + default = { + "roles/owner" = ["user:ludo@ludomagno.net"] + } +} + +variable "iam_roles" { + type = list(string) + default = ["roles/owner"] +} + +variable "key_iam_members" { + type = map(map(list(string))) + default = { + key-a = { + "roles/owner" = ["user:ludo@ludomagno.net"] + } + } +} + +variable "key_iam_roles" { + type = map(list(string)) + default = { + key-a = ["roles/owner"] + } +} + +variable "key_purpose" { + type = map(object({ + purpose = string + version_template = object({ + algorithm = string + protection_level = string + }) + })) + default = { + key-b = { + purpose = "ENCRYPT_DECRYPT" + version_template = null + } + key-c = { + purpose = "ASYMMETRIC_SIGN" + version_template = { + algorithm = "EC_SIGN_P384_SHA384" + protection_level = null + } + } + } +} + +variable "key_purpose_defaults" { + type = object({ + purpose = string + version_template = object({ + algorithm = string + protection_level = string + }) + }) + default = { + purpose = null + version_template = null + } +} + +variable "keyring" { + type = object({ + location = string + name = string + }) + default = { + location = "europe-west1" + name = "test-module" + } +} + +variable "keyring_create" { + type = bool + default = true +} + +variable "keys" { + type = map(object({ + rotation_period = string + labels = map(string) + })) + default = { + key-a = null + key-b = { rotation_period = "604800s", labels = null } + key-c = { rotation_period = null, labels = { env = "test" } } + } +} + +variable "project_id" { + type = string + default = "my-project" +} diff --git a/tests/modules/kms/test_plan.py b/tests/modules/kms/test_plan.py new file mode 100644 index 00000000..cd9d6c7c --- /dev/null +++ b/tests/modules/kms/test_plan.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + + +def test_resources(plan_runner): + "Test module resources." + _, resources = plan_runner(FIXTURES_DIR) + assert sorted(r['type'] for r in resources) == [ + 'google_kms_crypto_key', + 'google_kms_crypto_key', + 'google_kms_crypto_key', + 'google_kms_crypto_key_iam_binding', + 'google_kms_key_ring', + 'google_kms_key_ring_iam_binding' + ] diff --git a/tests/modules/net_vpc/test_plan.py b/tests/modules/net_vpc/test_plan.py index fc2852ca..23607ca7 100644 --- a/tests/modules/net_vpc/test_plan.py +++ b/tests/modules/net_vpc/test_plan.py @@ -86,5 +86,5 @@ def test_vpc_routes(plan_runner): _, resources = plan_runner(FIXTURES_DIR, routes=_var_routes) assert len(resources) == 3 resource = [r for r in resources if r['values'] - ['name'] == 'next-hop-test'][0] + ['name'] == 'my-vpc-next-hop-test'][0] assert resource['values']['next_hop_%s' % next_hop_type] diff --git a/tests/modules/organization/fixture/main.tf b/tests/modules/organization/fixture/main.tf index 20786b5e..63d1f466 100644 --- a/tests/modules/organization/fixture/main.tf +++ b/tests/modules/organization/fixture/main.tf @@ -20,8 +20,7 @@ module "test" { custom_roles = var.custom_roles iam_members = var.iam_members iam_roles = var.iam_roles - iam_additive_members = var.iam_additive_members - iam_additive_roles = var.iam_additive_roles + iam_additive_bindings= var.iam_additive_bindings iam_audit_config = var.iam_audit_config policy_boolean = var.policy_boolean policy_list = var.policy_list diff --git a/tests/modules/organization/fixture/variables.tf b/tests/modules/organization/fixture/variables.tf index 561b446c..148a43b7 100644 --- a/tests/modules/organization/fixture/variables.tf +++ b/tests/modules/organization/fixture/variables.tf @@ -29,15 +29,11 @@ variable "iam_roles" { default = [] } -variable "iam_additive_members" { +variable "iam_additive_bindings" { type = map(list(string)) default = {} } -variable "iam_additive_roles" { - type = list(string) - default = [] -} variable "iam_audit_config" { type = map(map(list(string))) diff --git a/tests/modules/project/fixture/main.tf b/tests/modules/project/fixture/main.tf index 8d8808c2..924b2648 100644 --- a/tests/modules/project/fixture/main.tf +++ b/tests/modules/project/fixture/main.tf @@ -22,8 +22,7 @@ module "test" { custom_roles = var.custom_roles iam_members = var.iam_members iam_roles = var.iam_roles - iam_additive_members = var.iam_additive_members - iam_additive_roles = var.iam_additive_roles + iam_additive_bindings = var.iam_additive_bindings labels = var.labels lien_reason = var.lien_reason oslogin = var.oslogin diff --git a/tests/modules/project/fixture/variables.tf b/tests/modules/project/fixture/variables.tf index 3c467da1..3b36a5fd 100644 --- a/tests/modules/project/fixture/variables.tf +++ b/tests/modules/project/fixture/variables.tf @@ -34,16 +34,11 @@ variable "iam_roles" { default = [] } -variable "iam_additive_members" { +variable "iam_additive_bindings" { type = map(list(string)) default = {} } -variable "iam_additive_roles" { - type = list(string) - default = [] -} - variable "labels" { type = map(string) default = {} @@ -71,7 +66,7 @@ variable "oslogin_users" { variable "parent" { type = string - default = "folders/12345678" + default = null } variable "policy_boolean" { diff --git a/tests/modules/project/test_plan.py b/tests/modules/project/test_plan.py index 4c5aba82..a526c1a5 100644 --- a/tests/modules/project/test_plan.py +++ b/tests/modules/project/test_plan.py @@ -32,7 +32,7 @@ def test_prefix(plan_runner): def test_parent(plan_runner): "Test project parent." - _, resources = plan_runner(FIXTURES_DIR) + _, resources = plan_runner(FIXTURES_DIR, parent='folders/12345678') assert len(resources) == 1 assert resources[0]['values']['folder_id'] == '12345678' assert resources[0]['values'].get('org_id') == None @@ -40,3 +40,11 @@ def test_parent(plan_runner): assert len(resources) == 1 assert resources[0]['values']['org_id'] == '12345678' assert resources[0]['values'].get('folder_id') == None + + +def test_no_parent(plan_runner): + "Test null project parent." + _, resources = plan_runner(FIXTURES_DIR) + assert len(resources) == 1 + assert resources[0]['values'].get('folder_id') == None + assert resources[0]['values'].get('org_id') == None