diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf
index 841bb803..b65989c3 100644
--- a/blueprints/cloud-operations/quota-monitoring/main.tf
+++ b/blueprints/cloud-operations/quota-monitoring/main.tf
@@ -49,6 +49,7 @@ module "pubsub" {
module "cf" {
source = "../../../modules/cloud-function-v1"
project_id = module.project.project_id
+ region = var.region
name = var.name
bucket_name = "${var.name}-${random_pet.random.id}"
bucket_config = {
diff --git a/blueprints/data-solutions/data-platform-minimal/02-composer.tf b/blueprints/data-solutions/data-platform-minimal/02-composer.tf
index 218df044..da6fca9a 100644
--- a/blueprints/data-solutions/data-platform-minimal/02-composer.tf
+++ b/blueprints/data-solutions/data-platform-minimal/02-composer.tf
@@ -109,6 +109,15 @@ resource "google_composer_environment" "processing-cmp-0" {
kms_key_name = var.service_encryption_keys.composer
}
}
+ web_server_network_access_control {
+ dynamic "allowed_ip_range" {
+ for_each = var.composer_config.web_server_access_control
+ content {
+ value = allowed_ip_range.key
+ description = allowed_ip_range.value
+ }
+ }
+ }
}
depends_on = [
module.processing-project
diff --git a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf
index 4275c559..3a68a7a8 100644
--- a/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf
+++ b/blueprints/data-solutions/data-platform-minimal/02-dataproc.tf
@@ -84,7 +84,7 @@ module "processing-dp-historyserver" {
staging_bucket = module.processing-staging-0.name
temp_bucket = module.processing-temp-0.name
gce_cluster_config = {
- subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link
+ subnetwork = local.processing_subnet
zone = "${var.region}-b"
service_account = module.processing-sa-0.email
service_account_scopes = ["cloud-platform"]
diff --git a/blueprints/data-solutions/data-platform-minimal/02-processing.tf b/blueprints/data-solutions/data-platform-minimal/02-processing.tf
index 53da3fa6..1d8cca2a 100644
--- a/blueprints/data-solutions/data-platform-minimal/02-processing.tf
+++ b/blueprints/data-solutions/data-platform-minimal/02-processing.tf
@@ -50,12 +50,12 @@ locals {
processing_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_link
- : module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"]
+ : try(module.processing-vpc.0.subnet_self_links["${var.region}/${var.prefix}-processing"], null)
)
processing_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
- : module.processing-vpc.0.self_link
+ : try(module.processing-vpc.0.self_link, null)
)
}
@@ -101,7 +101,7 @@ module "processing-project" {
host_project = var.network_config.host_project
service_identity_iam = {
"roles/compute.networkUser" = [
- "cloudservices", "compute", "container-engine", "dataflow"
+ "cloudservices", "compute", "container-engine", "dataflow", "dataproc"
]
"roles/composer.sharedVpcAgent" = [
"composer"
diff --git a/blueprints/data-solutions/data-platform-minimal/README.md b/blueprints/data-solutions/data-platform-minimal/README.md
index e459c37f..5760f3f6 100644
--- a/blueprints/data-solutions/data-platform-minimal/README.md
+++ b/blueprints/data-solutions/data-platform-minimal/README.md
@@ -69,7 +69,7 @@ We use three groups to control access to resources:
### Virtual Private Cloud (VPC) design
-As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project.
+As is often the case in real-world configurations, this blueprint accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable. Make sure that the GKE API (`container.googleapis.com`) is enabled in the VPC host project. Remember also to configure firewall rules needed for the different products you are going to use: Composer, Dataflow or Dataproc.
If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
@@ -280,13 +280,13 @@ The application layer is out of scope of this script. As a demo purpuse only, on
| [organization_domain](variables.tf#L119) | Organization domain. | string
| ✓ | |
| [prefix](variables.tf#L124) | Prefix used for resource names. | string
| ✓ | |
| [project_config](variables.tf#L133) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…})
| ✓ | |
-| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…})
| | {}
|
-| [data_catalog_tags](variables.tf#L54) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string)))
| | {…}
|
-| [data_force_destroy](variables.tf#L65) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool
| | false
|
-| [enable_services](variables.tf#L71) | Flag to enable or disable services in the Data Platform. | object({…})
| | {}
|
-| [groups](variables.tf#L80) | User groups. | map(string)
| | {…}
|
-| [location](variables.tf#L90) | Location used for multi-regional resources. | string
| | "eu"
|
-| [network_config](variables.tf#L96) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…})
| | {}
|
+| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…})
| | {}
|
+| [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string)))
| | {…}
|
+| [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool
| | false
|
+| [enable_services](variables.tf#L72) | Flag to enable or disable services in the Data Platform. | object({…})
| | {}
|
+| [groups](variables.tf#L81) | User groups. | map(string)
| | {…}
|
+| [location](variables.tf#L91) | Location used for multi-regional resources. | string
| | "eu"
|
+| [network_config](variables.tf#L97) | Shared VPC network configurations to use. If null networks will be created in projects. | object({…})
| | {}
|
| [project_suffix](variables.tf#L157) | Suffix used only for project ids. | string
| | null
|
| [region](variables.tf#L163) | Region used for regional resources. | string
| | "europe-west1"
|
| [service_encryption_keys](variables.tf#L169) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…})
| | {}
|
diff --git a/blueprints/data-solutions/data-platform-minimal/variables.tf b/blueprints/data-solutions/data-platform-minimal/variables.tf
index 09cdfdb8..a5f5143e 100644
--- a/blueprints/data-solutions/data-platform-minimal/variables.tf
+++ b/blueprints/data-solutions/data-platform-minimal/variables.tf
@@ -24,6 +24,7 @@ variable "composer_config" {
env_variables = optional(map(string), {})
image_version = optional(string, "composer-2-airflow-2")
}), {})
+ web_server_access_control = optional(map(string), {})
workloads_config = optional(object({
scheduler = optional(object({
cpu = optional(number, 0.5)
@@ -106,7 +107,6 @@ variable "network_config" {
pods_range_name = optional(string, "pods")
services_range_name = optional(string, "services")
}), {})
- # web_server_network_access_control = list(string)
})
nullable = false
default = {}
diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md
index 668f38f5..f848db57 100644
--- a/modules/dataproc/README.md
+++ b/modules/dataproc/README.md
@@ -145,16 +145,16 @@ module "processing-dp-cluster" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [name](variables.tf#L211) | Cluster name. | string
| ✓ | |
-| [project_id](variables.tf#L226) | Project ID. | string
| ✓ | |
-| [region](variables.tf#L231) | Dataproc region. | string
| ✓ | |
-| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…})
| | {}
|
-| [group_iam](variables.tf#L184) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string))
| | {}
|
-| [iam](variables.tf#L191) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
-| [iam_additive](variables.tf#L198) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
-| [labels](variables.tf#L205) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string)
| | {}
|
-| [prefix](variables.tf#L216) | Optional prefix used to generate project id and name. | string
| | null
|
-| [service_account](variables.tf#L236) | Service account to set on the Dataproc cluster. | string
| | null
|
+| [name](variables.tf#L212) | Cluster name. | string
| ✓ | |
+| [project_id](variables.tf#L227) | Project ID. | string
| ✓ | |
+| [region](variables.tf#L232) | Dataproc region. | string
| ✓ | |
+| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…})
| | {}
|
+| [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string))
| | {}
|
+| [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
+| [iam_additive](variables.tf#L199) | IAM additive bindings in {ROLE => [MEMBERS]} format. | map(list(string))
| | {}
|
+| [labels](variables.tf#L206) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string)
| | {}
|
+| [prefix](variables.tf#L217) | Optional prefix used to generate project id and name. | string
| | null
|
+| [service_account](variables.tf#L237) | Service account to set on the Dataproc cluster. | string
| | null
|
## Outputs
diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf
index 753a0523..926169b9 100644
--- a/modules/dataproc/variables.tf
+++ b/modules/dataproc/variables.tf
@@ -49,6 +49,7 @@ variable "dataproc_config" {
num_instances = number
machine_type = string
min_cpu_platform = string
+ image_uri = string
disk_config = optional(object({
boot_disk_type = string
boot_disk_size_gb = number