Merge branch 'master' into kthhrv-patch-1

blueprints/data-solutions/data-platform-minimal/02-dataproc.tf

@@ -84,7 +84,7 @@ module "processing-dp-historyserver" {
       staging_bucket = module.processing-staging-0.name
       temp_bucket    = module.processing-temp-0.name
       gce_cluster_config = {
-        subnetwork             = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link
+        subnetwork             = local.processing_subnet
         zone                   = "${var.region}-b"
         service_account        = module.processing-sa-0.email
         service_account_scopes = ["cloud-platform"]

blueprints/data-solutions/data-platform-minimal/02-processing.tf

@@ -50,12 +50,12 @@ locals {
   processing_subnet = (
     local.use_shared_vpc
     ? var.network_config.subnet_self_link
-    : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"]
+    : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null)
   )
   processing_vpc = (
     local.use_shared_vpc
     ? var.network_config.network_self_link
-    : module.processing-vpc.0.self_link
+    : try(module.processing-vpc.0.self_link, null)
   )
 }
 
@@ -101,7 +101,7 @@ module "processing-project" {
     host_project = var.network_config.host_project
     service_identity_iam = {
       "roles/compute.networkUser" = [
-        "cloudservices", "compute", "container-engine", "dataflow"
+        "cloudservices", "compute", "container-engine", "dataflow", "dataproc"
       ]
       "roles/composer.sharedVpcAgent" = [
         "composer"

blueprints/data-solutions/data-platform-minimal/README.md

@@ -69,7 +69,7 @@ We use three groups to control access to resources:
 
 ### Virtual Private Cloud (VPC) design
 
-As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project.
+As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc.
 
 If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
 

modules/dataproc/variables.tf

@@ -49,6 +49,7 @@ variable "dataproc_config" {
         num_instances    = number
         machine_type     = string
         min_cpu_platform = string
+        image_uri        = string
         disk_config = optional(object({
           boot_disk_type    = string
           boot_disk_size_gb = number