diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 841bb803..b65989c3 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -49,6 +49,7 @@ module "pubsub" { module "cf" { source = "../../../modules/cloud-function-v1" project_id = module.project.project_id + region = var.region name = var.name bucket_name = "${var.name}-${random_pet.random.id}" bucket_config = { diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf index 218df044..da6fca9a 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf @@ -109,6 +109,15 @@ resource "google_composer_environment" "processing-cmp-0" { kms_key_name = var.service_encryption_keys.composer } } + web_server_network_access_control { + dynamic "allowed_ip_range" { + for_each = var.composer_config.web_server_access_control + content { + value = allowed_ip_range.key + description = allowed_ip_range.value + } + } + } } depends_on = [ module.processing-project diff --git a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf index 4275c559..3a68a7a8 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf @@ -84,7 +84,7 @@ module "processing-dp-historyserver" { staging_bucket = module.processing-staging-0.name temp_bucket = module.processing-temp-0.name gce_cluster_config = { - subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link + subnetwork = local.processing_subnet zone = "${var.region}-b" service_account = module.processing-sa-0.email service_account_scopes = ["cloud-platform"] diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf index 53da3fa6..1d8cca2a 100644 --- a/blueprints/data-solutions/data-platform-minimal/02-processing.tf +++ b/blueprints/data-solutions/data-platform-minimal/02-processing.tf @@ -50,12 +50,12 @@ locals { processing_subnet = ( local.use_shared_vpc ? var.network_config.subnet_self_link - : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"] + : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null) ) processing_vpc = ( local.use_shared_vpc ? var.network_config.network_self_link - : module.processing-vpc.0.self_link + : try(module.processing-vpc.0.self_link, null) ) } @@ -101,7 +101,7 @@ module "processing-project" { host_project = var.network_config.host_project service_identity_iam = { "roles/compute.networkUser" = [ - "cloudservices", "compute", "container-engine", "dataflow" + "cloudservices", "compute", "container-engine", "dataflow", "dataproc" ] "roles/composer.sharedVpcAgent" = [ "composer" diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md index e459c37f..5760f3f6 100644 --- a/blueprints/data-solutions/data-platform-minimal/README.md +++ b/blueprints/data-solutions/data-platform-minimal/README.md @@ -69,7 +69,7 @@ We use three groups to control access to resources: ### Virtual Private Cloud (VPC) design -As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. +As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc. If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration). @@ -280,13 +280,13 @@ The application layer is out of scope of this script. As a demo purpuse only, on | [organization_domain](variables.tf#L119) | Organization domain. | string | ✓ | | | [prefix](variables.tf#L124) | Prefix used for resource names. | string | ✓ | | | [project_config](variables.tf#L133) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | -| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | -| [data_catalog_tags](variables.tf#L54) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | -| [data_force_destroy](variables.tf#L65) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [enable_services](variables.tf#L71) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | -| [groups](variables.tf#L80) | User groups. | map(string) | | {…} | -| [location](variables.tf#L90) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L96) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…}) | | {} | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {} | +| [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | +| [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [enable_services](variables.tf#L72) | Flag to enable or disable services in the Data Platform. | object({…}) | | {} | +| [groups](variables.tf#L81) | User groups. | map(string) | | {…} | +| [location](variables.tf#L91) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L97) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…}) | | {} | | [project_suffix](variables.tf#L157) | Suffix used only for project ids. | string | | null | | [region](variables.tf#L163) | Region used for regional resources. | string | | "europe-west1" | | [service_encryption_keys](variables.tf#L169) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | {} | diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf index 09cdfdb8..a5f5143e 100644 --- a/blueprints/data-solutions/data-platform-minimal/variables.tf +++ b/blueprints/data-solutions/data-platform-minimal/variables.tf @@ -24,6 +24,7 @@ variable "composer_config" { env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) + web_server_access_control = optional(map(string), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) @@ -106,7 +107,6 @@ variable "network_config" { pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) - # web_server_network_access_control = list(string) }) nullable = false default = {} diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 668f38f5..f848db57 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -145,16 +145,16 @@ module "processing-dp-cluster" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L211) | Cluster name. | string | ✓ | | -| [project_id](variables.tf#L226) | Project ID. | string | ✓ | | -| [region](variables.tf#L231) | Dataproc region. | string | ✓ | | -| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | -| [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | -| [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [labels](variables.tf#L205) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | -| [prefix](variables.tf#L216) | Optional prefix used to generate project id and name. | string | | null | -| [service_account](variables.tf#L236) | Service account to set on the Dataproc cluster. | string | | null | +| [name](variables.tf#L212) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L227) | Project ID. | string | ✓ | | +| [region](variables.tf#L232) | Dataproc region. | string | ✓ | | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_additive](variables.tf#L199) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L206) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L217) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L237) | Service account to set on the Dataproc cluster. | string | | null | ## Outputs diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 753a0523..926169b9 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -49,6 +49,7 @@ variable "dataproc_config" { num_instances = number machine_type = string min_cpu_platform = string + image_uri = string disk_config = optional(object({ boot_disk_type = string boot_disk_size_gb = number