From 9c26760773a1765e1278bd9a39acc048d1756d60 Mon Sep 17 00:00:00 2001 From: Jose Luis Bermudez <109610214+jose-bermudez-digitalfemsa@users.noreply.github.com> Date: Wed, 21 Jun 2023 16:19:48 -0600 Subject: [PATCH 01/12] add missing variable image_uri For master_config, the variable image_uri is missing --- modules/dataproc/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 753a0523..926169b9 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -49,6 +49,7 @@ variable "dataproc_config" { num_instances = number machine_type = string min_cpu_platform = string + image_uri = string disk_config = optional(object({ boot_disk_type = string boot_disk_size_gb = number From 7cd9083fd7ab47e46757a4945b5b65b188d0d02d Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 13:28:28 +0100 Subject: [PATCH 02/12] data-platform-minimal - support web_server_network_access_control --- .../data-solutions/data-platform-minimal/02-composer.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 218df044..09b3d859 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -109,6 +109,15 @@ resource "google_composer_environment" "processing-cmp-0" { kms_key_name = var.service_encryption_keys.composer } } + web_server_network_access_control { + dynamic "allowed_ip_range" { + for_each = var.webserver_access_ip_ranges + content { + value = allowed_ip_range.value["value"] + description = allowed_ip_range.value["description"] + } + } + } } depends_on = [ module.processing-project From b30efa8bfe7c06e5ace85ee8d2580a7ec3374929 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 13:56:00 +0100 Subject: [PATCH 03/12] Update variables.tf --- .../data-solutions/data-platform-minimal/variables.tf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index 09cdfdb8..40692b76 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,6 +24,16 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) + web_server_network_access_ip_ranges = optional(list(map(string)), [ + { + description = "Allows access from all IPv4 addresses (default value)" + value = "0.0.0.0/0" + }, + { + description = "Allows access from all IPv6 addresses (default value)" + value = "::0/0" + } + ]) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) @@ -106,7 +116,6 @@ variable "network_config" { pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) - # web_server_network_access_control = list(string) }) nullable = false default = {} From 9e8323bd7d6a7f4acc5a07f7b0b8d68189516fa2 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 13:56:35 +0100 Subject: [PATCH 04/12] Update 02-composer.tf --- blueprints/data-solutions/data-platform-minimal/02-composer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 09b3d859..4b6dbaff 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -111,7 +111,7 @@ resource "google_composer_environment" "processing-cmp-0" { } web_server_network_access_control { dynamic "allowed_ip_range" { - for_each = var.webserver_access_ip_ranges + for_each = var.composer_config.web_server_network_access_ip_ranges content { value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] From f9742c5405f1f6293af0d7b5b4b1eb0d691c1c9a Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 14:06:05 +0100 Subject: [PATCH 05/12] lint --- blueprints/data-solutions/data-platform-minimal/02-composer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 4b6dbaff..9ddb8a7b 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -113,7 +113,7 @@ resource "google_composer_environment" "processing-cmp-0" { dynamic "allowed_ip_range" { for_each = var.composer_config.web_server_network_access_ip_ranges content { - value = allowed_ip_range.value["value"] + value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] } } From 52ad1fee34d0b68e334ac3c99eb0247e7c40ef81 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Wed, 28 Jun 2023 14:07:36 +0100 Subject: [PATCH 06/12] more linting --- blueprints/data-solutions/data-platform-minimal/02-composer.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 9ddb8a7b..f4360226 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -111,7 +111,7 @@ resource "google_composer_environment" "processing-cmp-0" { } web_server_network_access_control { dynamic "allowed_ip_range" { - for_each = var.composer_config.web_server_network_access_ip_ranges + for_each = var.composer_config.web_server_network_access_ip_ranges content { value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] From 925a2883168754767ebb986c2d0481dbe3657478 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 28 Jun 2023 20:09:18 +0200 Subject: [PATCH 07/12] Update README --- modules/dataproc/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 668f38f5..f848db57 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -145,16 +145,16 @@ module "processing-dp-cluster" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L211) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L226) | Project ID. | string | ✓ | | -| [region](variables.tf#L231) | Dataproc region. | string | ✓ | | -| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | -| [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | -| [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [labels](variables.tf#L205) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | -| [prefix](variables.tf#L216) | Optional prefix used to generate project id and name. | string | | null | -| [service_account](variables.tf#L236) | Service account to set on the Dataproc cluster. | string | | null | +| [name](variables.tf#L212) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L227) | Project ID. | string | ✓ | | +| [region](variables.tf#L232) | Dataproc region. | string | ✓ | | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive](variables.tf#L199) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L206) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L217) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L237) | Service account to set on the Dataproc cluster. | string | | null | ## Outputs From 026071209cceadb98f39ea69bc10dfeac1934554 Mon Sep 17 00:00:00 2001 From: lcaggio Date: Wed, 28 Jun 2023 21:58:03 +0200 Subject: [PATCH 08/12] Minimal Data Platform - Shared VPC (#1475) * Fix * Fix dataproc vpc links * Add missing networkUser role. * Fix README. --- .../data-solutions/data-platform-minimal/02-dataproc.tf | 2 +- .../data-solutions/data-platform-minimal/02-processing.tf | 6 +++--- blueprints/data-solutions/data-platform-minimal/README.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf index 4275c559..3a68a7a8 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf @@ -84,7 +84,7 @@ module "processing-dp-historyserver" { staging_bucket = module.processing-staging-0.name temp_bucket = module.processing-temp-0.name gce_cluster_config = { - subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link + subnetwork = local.processing_subnet zone = "${var.region}-b" service_account = module.processing-sa-0.email service_account_scopes = ["cloud-platform"] diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf index 53da3fa6..1d8cca2a 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-processing.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-processing.tf @@ -50,12 +50,12 @@ locals { processing_subnet = ( local.use_shared_vpc ? var.network_config.subnet_self_link - : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"] + : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null) ) processing_vpc = ( local.use_shared_vpc ? var.network_config.network_self_link - : module.processing-vpc.0.self_link + : try(module.processing-vpc.0.self_link, null) ) } @@ -101,7 +101,7 @@ module "processing-project" { host_project = var.network_config.host_project service_identity_iam = { "roles/compute.networkUser" = [ - "cloudservices", "compute", "container-engine", "dataflow" + "cloudservices", "compute", "container-engine", "dataflow", "dataproc" ] "roles/composer.sharedVpcAgent" = [ "composer" diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index e459c37f..3d00ea49 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -69,7 +69,7 @@ We use three groups to control access to resources: ### Virtual Private Cloud (VPC) design -As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. +As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc. If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration). From e3e05540debc9bc765599dc01010a5b890bf0073 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Thu, 29 Jun 2023 09:45:29 +0100 Subject: [PATCH 09/12] change var name, drop default, update docs --- .../data-platform-minimal/02-composer.tf | 2 +- .../data-solutions/data-platform-minimal/README.md | 14 +++++++------- .../data-platform-minimal/variables.tf | 11 +---------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index f4360226..a890064b 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -111,7 +111,7 @@ resource "google_composer_environment" "processing-cmp-0" { } web_server_network_access_control { dynamic "allowed_ip_range" { - for_each = var.composer_config.web_server_network_access_ip_ranges + for_each = var.composer_config.web_server_access_control content { value = allowed_ip_range.value["value"] description = allowed_ip_range.value["description"] diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index e459c37f..f6ac475c 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -280,13 +280,13 @@ The application layer is out of scope of this script. As a demo purpuse only, on | [organization_domain](variables.tf#L119) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L124) | Prefix used for resource names. | string | ✓ | | | [project_config](variables.tf#L133) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | -| [data_catalog_tags](variables.tf#L54) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | -| [data_force_destroy](variables.tf#L65) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [enable_services](variables.tf#L71) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | -| [groups](variables.tf#L80) | User groups. | map(string) | | {…} | -| [location](variables.tf#L90) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L96) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…}) | | {} | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | +| [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | +| [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [enable_services](variables.tf#L72) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | +| [groups](variables.tf#L81) | User groups. | map(string) | | {…} | +| [location](variables.tf#L91) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L97) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…}) | | {} | | [project_suffix](variables.tf#L157) | Suffix used only for project ids. | string | | null | | [region](variables.tf#L163) | Region used for regional resources. | string | | "europe-west1" | | [service_encryption_keys](variables.tf#L169) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | {} | diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index 40692b76..afe57383 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,16 +24,7 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) - web_server_network_access_ip_ranges = optional(list(map(string)), [ - { - description = "Allows access from all IPv4 addresses (default value)" - value = "0.0.0.0/0" - }, - { - description = "Allows access from all IPv6 addresses (default value)" - value = "::0/0" - } - ]) + web_server_access_control = optional(map(string)) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) From f15e2f35092fbe5777e0b34b83db09d2f81fdac9 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Thu, 29 Jun 2023 09:55:10 +0100 Subject: [PATCH 10/12] default to empty map --- blueprints/data-solutions/data-platform-minimal/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index afe57383..a5f5143e 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,7 +24,7 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) - web_server_access_control = optional(map(string)) + web_server_access_control = optional(map(string), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) From 912497a158bb21b8ae559ae92e972c4ca26f5878 Mon Sep 17 00:00:00 2001 From: Keith Harvey Date: Thu, 29 Jun 2023 09:57:23 +0100 Subject: [PATCH 11/12] use key/value and update docs --- .../data-solutions/data-platform-minimal/02-composer.tf | 4 ++-- blueprints/data-solutions/data-platform-minimal/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index a890064b..da6fca9a 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -113,8 +113,8 @@ resource "google_composer_environment" "processing-cmp-0" { dynamic "allowed_ip_range" { for_each = var.composer_config.web_server_access_control content { - value = allowed_ip_range.value["value"] - description = allowed_ip_range.value["description"] + value = allowed_ip_range.key + description = allowed_ip_range.value } } } diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index f6ac475c..5411349b 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -280,7 +280,7 @@ The application layer is out of scope of this script. As a demo purpuse only, on | [organization_domain](variables.tf#L119) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L124) | Prefix used for resource names. | string | ✓ | | | [project_config](variables.tf#L133) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | | [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | | [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | | [enable_services](variables.tf#L72) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | From 981068e4428cb837b0d4455f0a592f224bcdf58d Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 29 Jun 2023 13:02:56 +0200 Subject: [PATCH 12/12] add region to cf (#1482) --- blueprints/cloud-operations/quota-monitoring/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 841bb803..b65989c3 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -49,6 +49,7 @@ module "pubsub" { module "cf" { source = "../../../modules/cloud-function-v1" project_id = module.project.project_id + region = var.region name = var.name bucket_name = "${var.name}-${random_pet.random.id}" bucket_config = {