2022-01-17 23:58:14 -08:00
|
|
|
# Copyright 2022 Google LLC
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# https://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2022-01-31 21:41:33 -08:00
|
|
|
# tfdoc:file:description Orchestration Cloud Composer definition
|
|
|
|
|
2022-01-17 23:58:14 -08:00
|
|
|
module "orc-sa-cmp-0" {
|
|
|
|
source = "../../../modules/iam-service-account"
|
|
|
|
project_id = module.orc-prj.project_id
|
|
|
|
name = "cmp-0"
|
|
|
|
prefix = local.prefix_orc
|
|
|
|
iam = {
|
|
|
|
"roles/iam.serviceAccountTokenCreator" = [
|
|
|
|
local.groups_iam.data-engineers
|
|
|
|
],
|
|
|
|
"roles/iam.serviceAccountUser" = [
|
|
|
|
module.orc-sa-cmp-0.iam_email,
|
|
|
|
]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_composer_environment" "orc-cmp-0" {
|
|
|
|
name = "${local.prefix_orc}-cmp-0"
|
|
|
|
region = var.composer_config.region
|
|
|
|
provider = google-beta
|
|
|
|
project = module.orc-prj.project_id
|
|
|
|
config {
|
|
|
|
node_count = 3
|
|
|
|
node_config {
|
|
|
|
zone = "${var.composer_config.region}-b"
|
|
|
|
service_account = module.orc-sa-cmp-0.email
|
|
|
|
network = module.orc-vpc[0].self_link
|
2022-01-31 08:11:06 -08:00
|
|
|
subnetwork = module.orc-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_orc}-subnet"]
|
2022-01-17 23:58:14 -08:00
|
|
|
tags = ["composer-worker", "http-server", "https-server"]
|
|
|
|
ip_allocation_policy {
|
|
|
|
use_ip_aliases = "true"
|
|
|
|
cluster_secondary_range_name = "pods"
|
|
|
|
services_secondary_range_name = "services"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
software_config {
|
2022-01-18 15:53:52 -08:00
|
|
|
image_version = "composer-1.17.5-airflow-2.1.4"
|
2022-01-17 23:58:14 -08:00
|
|
|
env_variables = {
|
2022-01-20 03:35:42 -08:00
|
|
|
DTL_EXP_PRJ = module.dtl-0-prj.project_id
|
2022-01-17 23:58:14 -08:00
|
|
|
DTL_EXP_BQ_DATASET = module.dtl-exp-bq-0.dataset_id
|
2022-01-20 03:35:42 -08:00
|
|
|
DTL_EXP_GCS = module.dtl-exp-cs-0.url
|
|
|
|
DTL_L0_PRJ = module.dtl-0-prj.project_id
|
|
|
|
DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id
|
2022-01-17 23:58:14 -08:00
|
|
|
DTL_L0_GCS = module.dtl-0-cs-0.url
|
2022-01-20 03:35:42 -08:00
|
|
|
DTL_L1_PRJ = module.dtl-1-prj.project_id
|
|
|
|
DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id
|
2022-01-17 23:58:14 -08:00
|
|
|
DTL_L1_GCS = module.dtl-1-cs-0.url
|
2022-01-20 03:35:42 -08:00
|
|
|
DTL_L2_PRJ = module.dtl-2-prj.project_id
|
|
|
|
DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id
|
2022-01-17 23:58:14 -08:00
|
|
|
DTL_L2_GCS = module.dtl-2-cs-0.url
|
2022-01-20 03:35:42 -08:00
|
|
|
GCP_REGION = var.composer_config.region
|
|
|
|
LND_PRJ = module.lnd-prj.project_id
|
2022-01-17 23:58:14 -08:00
|
|
|
LND_BQ = module.lnd-bq-0.dataset_id
|
2022-01-20 03:35:42 -08:00
|
|
|
LND_GCS = module.lnd-cs-0.url
|
2022-01-17 23:58:14 -08:00
|
|
|
LND_PS = module.lnd-ps-0.id
|
2022-01-20 03:35:42 -08:00
|
|
|
LOD_PRJ = module.lod-prj.project_id
|
2022-01-17 23:58:14 -08:00
|
|
|
LOD_GCS_STAGING = module.lod-cs-df-0.url
|
2022-01-31 09:58:55 -08:00
|
|
|
LOD_NET_VPC = module.lod-vpc[0].self_link
|
|
|
|
LOD_NET_SUBNET = module.lod-vpc[0].subnet_self_links["${var.composer_config.region}/${local.prefix_lod}-subnet"]
|
2022-01-20 03:35:42 -08:00
|
|
|
LOD_SA_DF = module.lod-sa-df-0.email
|
2022-01-17 23:58:14 -08:00
|
|
|
ORC_PRJ = module.orc-prj.project_id
|
2022-01-20 03:35:42 -08:00
|
|
|
ORC_GCS = module.orc-cs-0.url
|
2022-01-17 23:58:14 -08:00
|
|
|
TRF_PRJ = module.trf-prj.project_id
|
2022-01-20 03:35:42 -08:00
|
|
|
TRF_GCS_STAGING = module.trf-cs-df-0.url
|
2022-01-18 15:53:52 -08:00
|
|
|
TRF_SA_DF = module.trf-sa-df-0.email
|
|
|
|
TRF_SA_BQ = module.trf-sa-bq-0.email
|
2022-01-17 23:58:14 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
private_environment_config {
|
|
|
|
enable_private_endpoint = "true"
|
|
|
|
master_ipv4_cidr_block = var.composer_config.ip_range_gke_master
|
|
|
|
cloud_sql_ipv4_cidr_block = var.composer_config.ip_range_cloudsql
|
|
|
|
web_server_ipv4_cidr_block = var.composer_config.ip_range_web_server
|
|
|
|
}
|
|
|
|
|
|
|
|
dynamic "encryption_config" {
|
2022-01-31 08:11:06 -08:00
|
|
|
for_each = try(local.service_encryption_keys.composer != null, false) ? { 1 = 1 } : {}
|
2022-01-17 23:58:14 -08:00
|
|
|
content {
|
2022-01-31 08:11:06 -08:00
|
|
|
kms_key_name = try(local.service_encryption_keys.composer != null, false) ? try(local.service_encryption_keys.composer, null) : null
|
2022-01-17 23:58:14 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# web_server_network_access_control {
|
|
|
|
# allowed_ip_range {
|
|
|
|
# value = "172.16.0.0/12"
|
|
|
|
# description = "Allowed ip range"
|
|
|
|
# }
|
|
|
|
# }
|
|
|
|
}
|
|
|
|
depends_on = [
|
|
|
|
module.dtl-0-bq-0,
|
|
|
|
module.dtl-1-bq-0,
|
|
|
|
module.dtl-2-bq-0,
|
|
|
|
module.dtl-exp-bq-0,
|
|
|
|
module.dtl-0-cs-0,
|
|
|
|
module.dtl-1-cs-0,
|
|
|
|
module.dtl-2-cs-0,
|
|
|
|
module.dtl-exp-cs-0,
|
|
|
|
module.lnd-cs-0,
|
|
|
|
module.lnd-bq-0,
|
|
|
|
module.lnd-ps-0,
|
|
|
|
module.lod-cs-df-0,
|
|
|
|
module.trf-cs-df-0,
|
|
|
|
module.orc-vpc,
|
|
|
|
module.orc-vpc,
|
|
|
|
module.dtl-0-prj,
|
|
|
|
module.dtl-1-prj,
|
|
|
|
module.dtl-2-prj,
|
|
|
|
module.lnd-prj,
|
|
|
|
module.lod-prj,
|
|
|
|
module.orc-prj,
|
|
|
|
module.trf-prj,
|
|
|
|
module.lod-sa-df-0,
|
|
|
|
module.lod-sa-df-0,
|
|
|
|
]
|
|
|
|
}
|