mandatory project creation, refactor
This commit is contained in:
Ludovico Magnocavallo
parent
a64e7a8e41
commit
4f4a9cd7ac
|
|
@ -12,67 +12,40 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Landing project.
|
||||
# tfdoc:file:description land project and resources.
|
||||
|
||||
locals {
|
||||
group_iam_lnd = {
|
||||
"${local.groups.data-engineers}" = [
|
||||
land_orch_service_accounts = [
|
||||
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
}
|
||||
|
||||
module "land-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "lnd"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/pubsub.editor",
|
||||
"roles/storage.admin",
|
||||
"roles/storage.objectViewer",
|
||||
]
|
||||
}
|
||||
iam_lnd = {
|
||||
"roles/bigquery.dataEditor" = [
|
||||
module.lnd-sa-bq-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.dataViewer" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.orc-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/bigquery.user" = [
|
||||
module.lod-sa-df-0.iam_email
|
||||
]
|
||||
"roles/pubsub.publisher" = [
|
||||
module.lnd-sa-ps-0.iam_email
|
||||
]
|
||||
"roles/pubsub.subscriber" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/storage.objectCreator" = [
|
||||
module.lnd-sa-cs-0.iam_email,
|
||||
]
|
||||
"roles/storage.objectViewer" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/storage.admin" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
]
|
||||
iam = {
|
||||
"roles/bigquery.dataEditor" = [module.land-sa-bq-0.iam_email]
|
||||
"roles/bigquery.dataViewer" = local.land_orch_service_accounts
|
||||
"roles/bigquery.jobUser" = [module.orch-sa-cmp-0.iam_email]
|
||||
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
|
||||
"roles/pubsub.publisher" = [module.land-sa-ps-0.iam_email]
|
||||
"roles/pubsub.subscriber" = local.land_orch_service_accounts
|
||||
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
|
||||
"roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
|
||||
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
|
||||
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
|
||||
}
|
||||
prefix_lnd = "${var.prefix}-lnd"
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "lnd-prj" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["landing"], "lnd")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["landing"])
|
||||
prefix = can(var.project_ids["landing"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_lnd : {}
|
||||
iam_additive = var.project_create == null ? local.iam_lnd : {}
|
||||
group_iam = local.group_iam_lnd
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
@ -88,3 +61,73 @@ module "lnd-prj" {
|
|||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
}
|
||||
|
||||
# Cloud Storage
|
||||
|
||||
module "land-sa-cs-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.land-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "lnd-cs-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "land-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.land-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "lnd-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
# retention_policy = {
|
||||
# retention_period = 7776000 # 90 * 24 * 60 * 60
|
||||
# is_locked = false
|
||||
# }
|
||||
}
|
||||
|
||||
# PubSub
|
||||
|
||||
module "land-sa-ps-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.land-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "lnd-ps-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "land-ps-0" {
|
||||
source = "../../../modules/pubsub"
|
||||
project_id = module.land-project.project_id
|
||||
name = "${var.prefix}-lnd-ps-0"
|
||||
kms_key = try(local.service_encryption_keys.pubsub, null)
|
||||
}
|
||||
|
||||
# BigQuery
|
||||
|
||||
module "land-sa-bq-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.land-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "lnd-bq-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
|
||||
}
|
||||
}
|
||||
|
||||
module "land-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.land-project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}lnd_bq_0"
|
||||
location = var.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Landing storage resources (Bigquery, Cloud PubSub, Cloud Storage).
|
||||
|
||||
locals {
|
||||
lnd_bucket_retention_policy = {
|
||||
retention_period = 7776000 # 90 * 24 * 60 * 60
|
||||
is_locked = false
|
||||
}
|
||||
}
|
||||
|
||||
# Cloud Storage resources
|
||||
|
||||
module "lnd-sa-cs-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.lnd-prj.project_id
|
||||
name = "cs-0"
|
||||
prefix = local.prefix_lnd
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "lnd-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.lnd-prj.project_id
|
||||
name = "cs-0"
|
||||
prefix = local.prefix_lnd
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
# retention_policy = local.lnd_bucket_retention_policy
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
# Cloud PubSub resources
|
||||
|
||||
module "lnd-sa-ps-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.lnd-prj.project_id
|
||||
name = "ps-0"
|
||||
prefix = local.prefix_lnd
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "lnd-ps-0" {
|
||||
source = "../../../modules/pubsub"
|
||||
project_id = module.lnd-prj.project_id
|
||||
name = "${local.prefix_lnd}-ps-0"
|
||||
kms_key = try(local.service_encryption_keys.pubsub, null)
|
||||
}
|
||||
|
||||
# Bigquery resources
|
||||
|
||||
module "lnd-sa-bq-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.lnd-prj.project_id
|
||||
name = "bq-0"
|
||||
prefix = local.prefix_lnd
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "lnd-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.lnd-prj.project_id
|
||||
id = "${replace(local.prefix_lnd, "-", "_")}_bq_0"
|
||||
location = var.location_config.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
|
@ -15,52 +15,53 @@
|
|||
# tfdoc:file:description Load project and VPC.
|
||||
|
||||
locals {
|
||||
group_iam_lod = {
|
||||
"${local.groups.data-engineers}" = [
|
||||
load_service_accounts = [
|
||||
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
|
||||
module.load-sa-df-0.iam_email
|
||||
]
|
||||
load_subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.subnet_self_links.orchestration
|
||||
: values(module.load-vpc.0.subnet_self_links)[0]
|
||||
)
|
||||
load_vpc = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.network_self_link
|
||||
: module.load-vpc.0.self_link
|
||||
)
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "load-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "lod"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/compute.viewer",
|
||||
"roles/dataflow.admin",
|
||||
"roles/dataflow.developer",
|
||||
"roles/viewer",
|
||||
]
|
||||
}
|
||||
iam_lod = {
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.lod-sa-df-0.iam_email
|
||||
]
|
||||
"roles/compute.serviceAgent" = [
|
||||
"serviceAccount:${module.lod-prj.service_accounts.robots.compute}"
|
||||
]
|
||||
iam = {
|
||||
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
|
||||
"roles/dataflow.admin" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.lod-sa-df-0.iam_email
|
||||
]
|
||||
"roles/dataflow.worker" = [
|
||||
module.lod-sa-df-0.iam_email
|
||||
]
|
||||
"roles/dataflow.serviceAgent" = [
|
||||
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}"
|
||||
]
|
||||
"roles/storage.objectAdmin" = [
|
||||
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
|
||||
module.lod-sa-df-0.iam_email
|
||||
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
|
||||
]
|
||||
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
|
||||
"roles/storage.objectAdmin" = local.load_service_accounts
|
||||
# TODO: these are needed on the shared VPC?
|
||||
# "roles/compute.serviceAgent" = [
|
||||
# "serviceAccount:${module.load-project.service_accounts.robots.compute}"
|
||||
# ]
|
||||
# "roles/dataflow.serviceAgent" = [
|
||||
# "serviceAccount:${module.load-project.service_accounts.robots.dataflow}"
|
||||
# ]
|
||||
}
|
||||
prefix_lod = "${var.prefix}-lod"
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "lod-prj" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["load"], "lod")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["load"])
|
||||
prefix = can(var.project_ids["load"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_lod : {}
|
||||
iam_additive = var.project_create == null ? local.iam_lod : {}
|
||||
# group_iam = local.group_iam_lod
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
@ -79,37 +80,67 @@ module "lod-prj" {
|
|||
dataflow = [try(local.service_encryption_keys.dataflow, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
shared_vpc_service_config = local._shared_vpc_service_config
|
||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||
attach = true
|
||||
host_project = local.shared_vpc_project
|
||||
service_identity_iam = {
|
||||
# TODO: worker service account
|
||||
"compute.networkUser" = ["dataflow"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module "lod-vpc" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "load-sa-df-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.load-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "load-df-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
|
||||
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
|
||||
}
|
||||
}
|
||||
|
||||
module "load-cs-df-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.load-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "load-cs-0"
|
||||
storage_class = "REGIONAL"
|
||||
location = var.region
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
}
|
||||
|
||||
# internal VPC resources
|
||||
|
||||
module "load-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.lod-prj.project_id
|
||||
name = "${local.prefix_lod}-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.load-project.project_id
|
||||
name = "{var.prefix}-default"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.10.0.0/24"
|
||||
name = "${local.prefix_lod}-subnet"
|
||||
region = var.location_config.region
|
||||
name = "default"
|
||||
region = var.region
|
||||
secondary_ip_range = {}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
module "lod-vpc-firewall" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "load-vpc-firewall" {
|
||||
source = "../../../modules/net-vpc-firewall"
|
||||
project_id = module.lod-prj.project_id
|
||||
network = local._networks.load.network_name
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.load-project.project_id
|
||||
network = module.load-vpc.0.name
|
||||
admin_ranges = ["10.10.0.0/24"]
|
||||
}
|
||||
|
||||
module "lod-nat" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "load-nat" {
|
||||
source = "../../../modules/net-cloudnat"
|
||||
project_id = module.lod-prj.project_id
|
||||
region = var.location_config.region
|
||||
name = "${local.prefix_lod}-default"
|
||||
router_network = module.lod-vpc[0].name
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.load-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
region = var.region
|
||||
router_network = module.load-vpc.0.name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Load storage resources (Cloud Storage).
|
||||
|
||||
# Cloud Storage
|
||||
|
||||
module "lod-sa-df-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.lod-prj.project_id
|
||||
name = "df-0"
|
||||
prefix = local.prefix_lod
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "lod-cs-df-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.lod-prj.project_id
|
||||
name = "cs-0"
|
||||
prefix = local.prefix_lod
|
||||
storage_class = "REGIONAL"
|
||||
location = var.location_config.region
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
}
|
||||
|
|
@ -14,86 +14,96 @@
|
|||
|
||||
# tfdoc:file:description Orchestration Cloud Composer definition.
|
||||
|
||||
module "orc-sa-cmp-0" {
|
||||
module "orch-sa-cmp-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.orc-prj.project_id
|
||||
name = "cmp-0"
|
||||
prefix = local.prefix_orc
|
||||
project_id = module.orch-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "orc-cmp-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
|
||||
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_composer_environment" "orc-cmp-0" {
|
||||
name = "${local.prefix_orc}-cmp-0"
|
||||
region = var.location_config.region
|
||||
resource "google_composer_environment" "orch-cmp-0" {
|
||||
provider = google-beta
|
||||
project = module.orc-prj.project_id
|
||||
project = module.orch-project.project_id
|
||||
name = "${var.prefix}-orc-cmp-0"
|
||||
region = var.region
|
||||
config {
|
||||
node_count = var.composer_config.node_count
|
||||
node_config {
|
||||
zone = "${var.location_config.region}-b"
|
||||
service_account = module.orc-sa-cmp-0.email
|
||||
network = local._networks.orchestration.network
|
||||
subnetwork = local._networks.orchestration.subnet
|
||||
zone = "${var.region}-b"
|
||||
service_account = module.orch-sa-cmp-0.email
|
||||
network = local.orch_vpc
|
||||
subnetwork = local.orch_subnet
|
||||
tags = ["composer-worker", "http-server", "https-server"]
|
||||
ip_allocation_policy {
|
||||
use_ip_aliases = "true"
|
||||
cluster_secondary_range_name = try(var.network_config.composer_secondary_ranges.pods, "pods")
|
||||
services_secondary_range_name = try(var.network_config.composer_secondary_ranges.services, "services")
|
||||
use_ip_aliases = "true"
|
||||
cluster_secondary_range_name = try(
|
||||
var.network_config.composer_secondary_ranges.pods, "pods"
|
||||
)
|
||||
services_secondary_range_name = try(
|
||||
var.network_config.composer_secondary_ranges.services, "services"
|
||||
)
|
||||
}
|
||||
}
|
||||
software_config {
|
||||
image_version = var.composer_config.airflow_version
|
||||
env_variables = merge(
|
||||
var.composer_config.env_variables, {
|
||||
DTL_L0_PRJ = module.dtl-0-prj.project_id
|
||||
DTL_L0_BQ_DATASET = module.dtl-0-bq-0.dataset_id
|
||||
DTL_L0_GCS = module.dtl-0-cs-0.url
|
||||
DTL_L1_PRJ = module.dtl-1-prj.project_id
|
||||
DTL_L1_BQ_DATASET = module.dtl-1-bq-0.dataset_id
|
||||
DTL_L1_GCS = module.dtl-1-cs-0.url
|
||||
DTL_L2_PRJ = module.dtl-2-prj.project_id
|
||||
DTL_L2_BQ_DATASET = module.dtl-2-bq-0.dataset_id
|
||||
DTL_L2_GCS = module.dtl-2-cs-0.url
|
||||
DTL_PLG_PRJ = module.dtl-plg-prj.project_id
|
||||
DTL_PLG_BQ_DATASET = module.dtl-plg-bq-0.dataset_id
|
||||
DTL_PLG_GCS = module.dtl-plg-cs-0.url
|
||||
GCP_REGION = var.location_config.region
|
||||
LND_PRJ = module.lnd-prj.project_id
|
||||
LND_BQ = module.lnd-bq-0.dataset_id
|
||||
LND_GCS = module.lnd-cs-0.url
|
||||
LND_PS = module.lnd-ps-0.id
|
||||
LOD_PRJ = module.lod-prj.project_id
|
||||
LOD_GCS_STAGING = module.lod-cs-df-0.url
|
||||
LOD_NET_VPC = local._networks.load.network
|
||||
LOD_NET_SUBNET = local._networks.load.subnet
|
||||
LOD_SA_DF = module.lod-sa-df-0.email
|
||||
ORC_PRJ = module.orc-prj.project_id
|
||||
ORC_GCS = module.orc-cs-0.url
|
||||
TRF_PRJ = module.trf-prj.project_id
|
||||
TRF_GCS_STAGING = module.trf-cs-df-0.url
|
||||
TRF_NET_VPC = local._networks.transformation.network
|
||||
TRF_NET_SUBNET = local._networks.transformation.subnet
|
||||
TRF_SA_DF = module.trf-sa-df-0.email
|
||||
TRF_SA_BQ = module.trf-sa-bq-0.email
|
||||
DTL_L0_PRJ = module.lake-0-project.project_id
|
||||
DTL_L0_BQ_DATASET = module.lake-0-bq-0.dataset_id
|
||||
DTL_L0_GCS = module.lake-0-cs-0.url
|
||||
DTL_L1_PRJ = module.lake-1-project.project_id
|
||||
DTL_L1_BQ_DATASET = module.lake-1-bq-0.dataset_id
|
||||
DTL_L1_GCS = module.lake-1-cs-0.url
|
||||
DTL_L2_PRJ = module.lake-2-project.project_id
|
||||
DTL_L2_BQ_DATASET = module.lake-2-bq-0.dataset_id
|
||||
DTL_L2_GCS = module.lake-2-cs-0.url
|
||||
DTL_PLG_PRJ = module.lake-plg-project.project_id
|
||||
DTL_PLG_BQ_DATASET = module.lake-plg-bq-0.dataset_id
|
||||
DTL_PLG_GCS = module.lake-plg-cs-0.url
|
||||
GCP_REGION = var.region
|
||||
LND_PRJ = module.land-project.project_id
|
||||
LND_BQ = module.land-bq-0.dataset_id
|
||||
LND_GCS = module.land-cs-0.url
|
||||
LND_PS = module.land-ps-0.id
|
||||
LOD_PRJ = module.load-project.project_id
|
||||
LOD_GCS_STAGING = module.load-cs-df-0.url
|
||||
LOD_NET_VPC = local.load_vpc
|
||||
LOD_NET_SUBNET = local.load_subnet
|
||||
LOD_SA_DF = module.load-sa-df-0.email
|
||||
ORC_PRJ = module.orch-project.project_id
|
||||
ORC_GCS = module.orch-cs-0.url
|
||||
TRF_PRJ = module.transf-project.project_id
|
||||
TRF_GCS_STAGING = module.transf-cs-df-0.url
|
||||
TRF_NET_VPC = local.transf_vpc
|
||||
TRF_NET_SUBNET = local.transf_subnet
|
||||
TRF_SA_DF = module.transf-sa-df-0.email
|
||||
TRF_SA_BQ = module.transf-sa-bq-0.email
|
||||
}
|
||||
)
|
||||
}
|
||||
private_environment_config {
|
||||
enable_private_endpoint = "true"
|
||||
cloud_sql_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24")
|
||||
master_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28")
|
||||
web_server_ipv4_cidr_block = try(var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28")
|
||||
enable_private_endpoint = "true"
|
||||
cloud_sql_ipv4_cidr_block = try(
|
||||
var.network_config.composer_ip_ranges.cloudsql, "10.20.10.0/24"
|
||||
)
|
||||
master_ipv4_cidr_block = try(
|
||||
var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28"
|
||||
)
|
||||
web_server_ipv4_cidr_block = try(
|
||||
var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28"
|
||||
)
|
||||
}
|
||||
|
||||
dynamic "encryption_config" {
|
||||
for_each = try(local.service_encryption_keys.composer != null, false) ? { 1 = 1 } : {}
|
||||
for_each = (
|
||||
try(local.service_encryption_keys.composer != null, false)
|
||||
? { 1 = 1 }
|
||||
: {}
|
||||
)
|
||||
content {
|
||||
kms_key_name = try(local.service_encryption_keys.composer, null)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,8 +15,26 @@
|
|||
# tfdoc:file:description Orchestration project and VPC.
|
||||
|
||||
locals {
|
||||
group_iam_orc = {
|
||||
"${local.groups.data-engineers}" = [
|
||||
orch_subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.subnet_self_links.orchestration
|
||||
: values(module.orch-vpc.0.subnet_self_links)[0]
|
||||
)
|
||||
orch_vpc = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.network_self_link
|
||||
: module.orch-vpc.0.self_link
|
||||
)
|
||||
}
|
||||
|
||||
module "orch-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "orc"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/cloudbuild.builds.editor",
|
||||
|
|
@ -30,98 +48,99 @@ locals {
|
|||
"roles/compute.networkUser"
|
||||
]
|
||||
}
|
||||
|
||||
iam_orc = {
|
||||
iam = {
|
||||
"roles/bigquery.dataEditor" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/composer.worker" = [
|
||||
module.orc-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/compute.networkUser" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
|
||||
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
|
||||
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
|
||||
"serviceAccount:${module.orc-prj.service_accounts.cloud_services}"
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
"serviceAccount:${module.orc-prj.service_accounts.robots.composer}",
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
|
||||
]
|
||||
"roles/storage.admin" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email
|
||||
]
|
||||
}
|
||||
|
||||
prefix_orc = "${var.prefix}-orc"
|
||||
}
|
||||
|
||||
module "orc-prj" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["orchestration"], "orc")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["orchestration"])
|
||||
prefix = can(var.project_ids["orchestration"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_orc : {}
|
||||
iam_additive = var.project_create == null ? local.iam_orc : {}
|
||||
group_iam = local.group_iam_orc
|
||||
oslogin = false
|
||||
oslogin = false
|
||||
policy_boolean = {
|
||||
"constraints/compute.requireOsLogin" = false
|
||||
}
|
||||
services = concat(
|
||||
var.project_services,
|
||||
[
|
||||
"artifactregistry.googleapis.com",
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudbuild.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"composer.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"container.googleapis.com",
|
||||
"containerregistry.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
services = concat(var.project_services, [
|
||||
"artifactregistry.googleapis.com",
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudbuild.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"composer.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"container.googleapis.com",
|
||||
"containerregistry.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
])
|
||||
service_encryption_key_ids = {
|
||||
composer = [try(local.service_encryption_keys.composer, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
shared_vpc_service_config = local._shared_vpc_service_config
|
||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||
attach = true
|
||||
host_project = local.shared_vpc_project
|
||||
service_identity_iam = {
|
||||
"roles/composer.sharedVpcAgent" = [
|
||||
"composer"
|
||||
]
|
||||
"roles/compute.networkUser" = [
|
||||
"cloudservices", "container-engine", "dataflow"
|
||||
]
|
||||
"roles/container.hostServiceAgentUser" = [
|
||||
"container-engine"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module "orc-vpc" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
# Cloud Storage
|
||||
|
||||
module "orch-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.orch-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "orc-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
}
|
||||
|
||||
# internal VPC resources
|
||||
|
||||
module "orch-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.orc-prj.project_id
|
||||
name = "${local.prefix_orc}-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.orch-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.10.0.0/24"
|
||||
name = "${local.prefix_orc}-subnet"
|
||||
region = var.location_config.region
|
||||
name = "default"
|
||||
region = var.region
|
||||
secondary_ip_range = {
|
||||
pods = "10.10.8.0/22"
|
||||
services = "10.10.12.0/24"
|
||||
|
|
@ -130,52 +149,19 @@ module "orc-vpc" {
|
|||
]
|
||||
}
|
||||
|
||||
resource "google_project_iam_binding" "composer_shared_vpc_agent" {
|
||||
count = var.network_config.network_self_link != null ? 1 : 0
|
||||
project = local._shared_vpc_project
|
||||
role = "roles/composer.sharedVpcAgent"
|
||||
members = [
|
||||
"serviceAccount:${module.orc-prj.service_accounts.robots.composer}"
|
||||
]
|
||||
}
|
||||
|
||||
resource "google_project_iam_binding" "gke_host_service_agent_user" {
|
||||
count = var.network_config.network_self_link != null ? 1 : 0
|
||||
project = local._shared_vpc_project
|
||||
role = "roles/container.hostServiceAgentUser"
|
||||
members = [
|
||||
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}"
|
||||
]
|
||||
}
|
||||
|
||||
resource "google_project_iam_binding" "composer_network_user_agent" {
|
||||
count = var.network_config.network_self_link != null ? 1 : 0
|
||||
project = local._shared_vpc_project
|
||||
role = "roles/compute.networkUser"
|
||||
members = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
"serviceAccount:${module.lod-prj.service_accounts.robots.dataflow}",
|
||||
"serviceAccount:${module.orc-prj.service_accounts.cloud_services}",
|
||||
"serviceAccount:${module.orc-prj.service_accounts.robots.container-engine}",
|
||||
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}",
|
||||
]
|
||||
}
|
||||
|
||||
module "orc-vpc-firewall" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "orch-vpc-firewall" {
|
||||
source = "../../../modules/net-vpc-firewall"
|
||||
project_id = module.orc-prj.project_id
|
||||
network = local._networks.orchestration.network_name
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.orch-project.project_id
|
||||
network = module.orch-vpc.0.name
|
||||
admin_ranges = ["10.10.0.0/24"]
|
||||
}
|
||||
|
||||
module "orc-nat" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "orch-nat" {
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
source = "../../../modules/net-cloudnat"
|
||||
project_id = module.orc-prj.project_id
|
||||
region = var.location_config.region
|
||||
name = "${local.prefix_orc}-default"
|
||||
router_network = local._networks.orchestration.network_name
|
||||
project_id = module.orch-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
region = var.region
|
||||
router_network = module.orch-vpc.0.name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Orchestration storage resources (Cloud Storage).
|
||||
|
||||
# Cloud Storage
|
||||
|
||||
module "orc-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.orc-prj.project_id
|
||||
name = "cs-0"
|
||||
prefix = local.prefix_orc
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Landing storage resources (Bigquery, Cloud Storage).
|
||||
|
||||
# Cloud Storage
|
||||
|
||||
module "trf-sa-df-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.trf-prj.project_id
|
||||
name = "df-0"
|
||||
prefix = local.prefix_trf
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "trf-cs-df-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.trf-prj.project_id
|
||||
name = "cs-0"
|
||||
prefix = local.prefix_trf
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
}
|
||||
|
||||
# Bigquery
|
||||
|
||||
module "trf-sa-bq-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.trf-prj.project_id
|
||||
name = "bq-0"
|
||||
prefix = local.prefix_trf
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -15,47 +15,49 @@
|
|||
# tfdoc:file:description Trasformation project and VPC.
|
||||
|
||||
locals {
|
||||
group_iam_trf = {
|
||||
"${local.groups.data-engineers}" = [
|
||||
transf_subnet = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.subnet_self_links.orchestration
|
||||
: values(module.transf-vpc.0.subnet_self_links)[0]
|
||||
)
|
||||
transf_vpc = (
|
||||
local.use_shared_vpc
|
||||
? var.network_config.network_self_link
|
||||
: module.transf-vpc.0.self_link
|
||||
)
|
||||
}
|
||||
|
||||
module "transf-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "trf"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/dataflow.admin",
|
||||
]
|
||||
}
|
||||
iam_trf = {
|
||||
iam = {
|
||||
"roles/bigquery.dataViewer" = [
|
||||
module.orc-sa-cmp-0.iam_email
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.trf-sa-bq-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
]
|
||||
"roles/dataflow.admin" = [
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/dataflow.worker" = [
|
||||
module.trf-sa-df-0.iam_email
|
||||
module.transf-sa-df-0.iam_email
|
||||
]
|
||||
"roles/storage.objectAdmin" = [
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
"serviceAccount:${module.trf-prj.service_accounts.robots.dataflow}"
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
|
||||
]
|
||||
}
|
||||
prefix_trf = "${var.prefix}-trf"
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "trf-prj" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["trasformation"], "trf")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["trasformation"])
|
||||
prefix = can(var.project_ids["trasformation"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_trf : {}
|
||||
iam_additive = var.project_create == null ? local.iam_trf : {}
|
||||
group_iam = local.group_iam_trf
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
|
|
@ -73,37 +75,89 @@ module "trf-prj" {
|
|||
dataflow = [try(local.service_encryption_keys.dataflow, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
shared_vpc_service_config = local._shared_vpc_service_config
|
||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||
attach = true
|
||||
host_project = local.shared_vpc_project
|
||||
service_identity_iam = {}
|
||||
}
|
||||
}
|
||||
|
||||
module "trf-vpc" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
# Cloud Storage
|
||||
|
||||
module "transf-sa-df-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.transf-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "trf-df-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers,
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module "transf-cs-df-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.transf-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "trf-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
}
|
||||
|
||||
# BigQuery
|
||||
|
||||
module "transf-sa-bq-0" {
|
||||
source = "../../../modules/iam-service-account"
|
||||
project_id = module.transf-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "trf-bq-0"
|
||||
iam = {
|
||||
"roles/iam.serviceAccountTokenCreator" = [
|
||||
local.groups_iam.data-engineers,
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
],
|
||||
"roles/iam.serviceAccountUser" = [
|
||||
module.orch-sa-cmp-0.iam_email
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
# internal VPC resources
|
||||
|
||||
module "transf-vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
project_id = module.trf-prj.project_id
|
||||
name = "${local.prefix_trf}-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.transf-project.project_id
|
||||
name = "{var.prefix}-default"
|
||||
subnets = [
|
||||
{
|
||||
ip_cidr_range = "10.10.0.0/24"
|
||||
name = "${local.prefix_trf}-subnet"
|
||||
region = var.location_config.region
|
||||
name = "default"
|
||||
region = var.region
|
||||
secondary_ip_range = {}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
module "trf-vpc-firewall" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "transf-vpc-firewall" {
|
||||
source = "../../../modules/net-vpc-firewall"
|
||||
project_id = module.trf-prj.project_id
|
||||
network = local._networks.transformation.network_name
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.transf-project.project_id
|
||||
network = module.transf-vpc.0.name
|
||||
admin_ranges = ["10.10.0.0/24"]
|
||||
}
|
||||
|
||||
module "trf-nat" {
|
||||
count = var.network_config.network_self_link != null ? 0 : 1
|
||||
module "transf-nat" {
|
||||
source = "../../../modules/net-cloudnat"
|
||||
project_id = module.trf-prj.project_id
|
||||
region = var.location_config.region
|
||||
name = "${local.prefix_trf}-default"
|
||||
router_network = module.trf-vpc[0].name
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.transf-project.project_id
|
||||
name = "${var.prefix}-default"
|
||||
region = var.region
|
||||
router_network = module.transf-vpc.0.name
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,12 +15,12 @@
|
|||
# tfdoc:file:description Datalake projects.
|
||||
|
||||
locals {
|
||||
group_iam_dtl = {
|
||||
"${local.groups.data-engineers}" = [
|
||||
lake_group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/bigquery.dataEditor",
|
||||
"roles/storage.admin",
|
||||
],
|
||||
"${local.groups.data-analysts}" = [
|
||||
(local.groups.data-analysts) = [
|
||||
"roles/bigquery.dataViewer",
|
||||
"roles/bigquery.jobUser",
|
||||
"roles/bigquery.user",
|
||||
|
|
@ -29,150 +29,185 @@ locals {
|
|||
"roles/storage.objectViewer",
|
||||
]
|
||||
}
|
||||
iam_dtl = {
|
||||
lake_iam = {
|
||||
"roles/bigquery.dataEditor" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.trf-sa-bq-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/bigquery.jobUser" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/storage.admin" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
]
|
||||
"roles/storage.objectCreator" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.trf-sa-bq-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
"roles/storage.objectViewer" = [
|
||||
module.trf-sa-df-0.iam_email,
|
||||
module.trf-sa-bq-0.iam_email,
|
||||
module.orc-sa-cmp-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email,
|
||||
module.transf-sa-bq-0.iam_email,
|
||||
module.orch-sa-cmp-0.iam_email,
|
||||
]
|
||||
}
|
||||
prefix_dtl = "${var.prefix}-dtl"
|
||||
lake_services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
])
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "dtl-0-prj" {
|
||||
module "lake-0-project" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["datalake-l0"], "dtl-0")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["datalake-l0"])
|
||||
prefix = can(var.project_ids["datalake-l0"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_dtl : {}
|
||||
iam_additive = var.project_create == null ? local.iam_dtl : {}
|
||||
group_iam = local.group_iam_dtl
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
])
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-0"
|
||||
group_iam = local.lake_group_iam
|
||||
iam = local.lake_iam
|
||||
services = local.lake_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
}
|
||||
|
||||
module "dtl-1-prj" {
|
||||
module "lake-1-project" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["datalake-l1"], "dtl-1")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["datalake-l1"])
|
||||
prefix = can(var.project_ids["datalake-l1"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_dtl : {}
|
||||
iam_additive = var.project_create == null ? local.iam_dtl : {}
|
||||
group_iam = local.group_iam_dtl
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
])
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-1"
|
||||
group_iam = local.lake_group_iam
|
||||
iam = local.lake_iam
|
||||
services = local.lake_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
}
|
||||
|
||||
module "dtl-2-prj" {
|
||||
module "lake-2-project" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["datalake-l2"], "dtl-2")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["datalake-l2"])
|
||||
prefix = can(var.project_ids["datalake-l2"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_dtl : {}
|
||||
iam_additive = var.project_create == null ? local.iam_dtl : {}
|
||||
group_iam = local.group_iam_dtl
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
])
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-2"
|
||||
group_iam = local.lake_group_iam
|
||||
iam = local.lake_iam
|
||||
services = local.lake_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
}
|
||||
|
||||
module "dtl-plg-prj" {
|
||||
module "lake-plg-project" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["datalake-playground"], "dtl-plg")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["datalake-playground"])
|
||||
prefix = can(var.project_ids["datalake-playground"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_dtl : {}
|
||||
iam_additive = var.project_create == null ? local.iam_dtl : {}
|
||||
group_iam = local.group_iam_dtl
|
||||
services = concat(var.project_services, [
|
||||
"bigquery.googleapis.com",
|
||||
"bigqueryreservation.googleapis.com",
|
||||
"bigquerystorage.googleapis.com",
|
||||
"cloudkms.googleapis.com",
|
||||
"compute.googleapis.com",
|
||||
"dataflow.googleapis.com",
|
||||
"pubsub.googleapis.com",
|
||||
"servicenetworking.googleapis.com",
|
||||
"storage.googleapis.com",
|
||||
"storage-component.googleapis.com"
|
||||
])
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-plg"
|
||||
group_iam = local.lake_group_iam
|
||||
iam = local.lake_iam
|
||||
services = local.lake_services
|
||||
service_encryption_key_ids = {
|
||||
bq = [try(local.service_encryption_keys.bq, null)]
|
||||
storage = [try(local.service_encryption_keys.storage, null)]
|
||||
}
|
||||
}
|
||||
|
||||
# Bigquery
|
||||
|
||||
module "lake-0-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.lake-0-project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}_dtl_0_bq_0"
|
||||
location = var.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
module "lake-1-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.lake-1-project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}_dtl_1_bq_0"
|
||||
location = var.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
module "lake-2-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.lake-2-project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}_dtl_2_bq_0"
|
||||
location = var.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
module "lake-plg-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.lake-plg-project.project_id
|
||||
id = "${replace(var.prefix, "-", "_")}_dtl_plg_bq_0"
|
||||
location = var.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
# Cloud storage
|
||||
|
||||
module "lake-0-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.lake-0-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-0-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
module "lake-1-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.lake-1-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-1-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
module "lake-2-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.lake-2-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-2-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
module "lake-plg-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.lake-plg-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "dtl-plg-cs-0"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,95 +0,0 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# tfdoc:file:description Datalake storage resources (Bigquery, Cloud Storage).
|
||||
|
||||
# Bigquery
|
||||
|
||||
module "dtl-0-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.dtl-0-prj.project_id
|
||||
id = "${replace(local.prefix_dtl, "-", "_")}_0_bq_0"
|
||||
location = var.location_config.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
module "dtl-1-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.dtl-1-prj.project_id
|
||||
id = "${replace(local.prefix_dtl, "-", "_")}_1_bq_0"
|
||||
location = var.location_config.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
module "dtl-2-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.dtl-2-prj.project_id
|
||||
id = "${replace(local.prefix_dtl, "-", "_")}_2_bq_0"
|
||||
location = var.location_config.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
module "dtl-plg-bq-0" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
project_id = module.dtl-plg-prj.project_id
|
||||
id = "${replace(local.prefix_dtl, "-", "_")}_plg_bq_0"
|
||||
location = var.location_config.region
|
||||
encryption_key = try(local.service_encryption_keys.bq, null)
|
||||
}
|
||||
|
||||
# Cloud storage
|
||||
|
||||
module "dtl-0-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.dtl-0-prj.project_id
|
||||
name = "0-cs-0"
|
||||
prefix = local.prefix_dtl
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
module "dtl-1-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.dtl-1-prj.project_id
|
||||
name = "1-cs-0"
|
||||
prefix = local.prefix_dtl
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
module "dtl-2-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.dtl-2-prj.project_id
|
||||
name = "2-cs-0"
|
||||
prefix = local.prefix_dtl
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
||||
module "dtl-plg-cs-0" {
|
||||
source = "../../../modules/gcs"
|
||||
project_id = module.dtl-plg-prj.project_id
|
||||
name = "plg-cs-0"
|
||||
prefix = local.prefix_dtl
|
||||
location = var.location_config.region
|
||||
storage_class = "REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = var.data_force_destroy
|
||||
}
|
||||
|
|
@ -14,39 +14,28 @@
|
|||
|
||||
# tfdoc:file:description common project.
|
||||
|
||||
locals {
|
||||
group_iam_cmn = {
|
||||
"${local.groups.data-engineers}" = [
|
||||
module "common-project" {
|
||||
source = "../../../modules/project"
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "cmn"
|
||||
group_iam = {
|
||||
(local.groups.data-engineers) = [
|
||||
"roles/dlp.reader",
|
||||
"roles/dlp.user",
|
||||
"roles/dlp.estimatesAdmin",
|
||||
],
|
||||
"${local.groups.data-security}" = [
|
||||
]
|
||||
(local.groups.data-security) = [
|
||||
"roles/dlp.admin",
|
||||
],
|
||||
}
|
||||
iam_cmn = {
|
||||
"roles/dlp.user" = [
|
||||
module.lod-sa-df-0.iam_email,
|
||||
module.trf-sa-df-0.iam_email
|
||||
]
|
||||
}
|
||||
prefix_cmn = "${var.prefix}-cmn"
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "cmn-prj" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["common"], "cmn")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["common"])
|
||||
prefix = can(var.project_ids["common"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_cmn : {}
|
||||
iam_additive = var.project_create == null ? local.iam_cmn : {}
|
||||
group_iam = local.group_iam_cmn
|
||||
iam = {
|
||||
"roles/dlp.user" = [
|
||||
module.load-sa-df-0.iam_email,
|
||||
module.transf-sa-df-0.iam_email
|
||||
]
|
||||
}
|
||||
services = concat(var.project_services, [
|
||||
"datacatalog.googleapis.com",
|
||||
"dlp.googleapis.com",
|
||||
|
|
|
|||
|
|
@ -14,29 +14,10 @@
|
|||
|
||||
# tfdoc:file:description common project.
|
||||
|
||||
locals {
|
||||
group_iam_exp = {
|
||||
#TODO add group => role mapping to asign on exposure project
|
||||
}
|
||||
iam_exp = {
|
||||
#TODO add role => service account mapping to assign roles on exposure project
|
||||
}
|
||||
prefix_exp = "${var.prefix}-exp"
|
||||
}
|
||||
|
||||
# Project
|
||||
|
||||
module "exp-prj" {
|
||||
module "exp-project" {
|
||||
source = "../../../modules/project"
|
||||
name = try(var.project_ids["exposure"], "exp")
|
||||
parent = try(var.project_create.parent, null)
|
||||
billing_account = try(var.project_create.billing_account_id, null)
|
||||
project_create = can(var.project_ids["exposure"])
|
||||
prefix = can(var.project_ids["exposure"]) ? var.prefix : null
|
||||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam_exp : {}
|
||||
iam_additive = var.project_create == null ? local.iam_exp : {}
|
||||
group_iam = local.group_iam_exp
|
||||
services = concat(var.project_services, [
|
||||
])
|
||||
parent = var.folder_id
|
||||
billing_account = var.billing_account_id
|
||||
prefix = var.prefix
|
||||
name = "exp"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -241,16 +241,16 @@ Description of commands:
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [network_config](variables.tf#L47) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string web_server = string }) composer_secondary_ranges = object({ pods = string services = string }) }) default = { network_self_link = null subnet_self_links = null composer_ip_ranges = null composer_secondary_ranges = null }">object({…}</code> | ✓ | |
|
||||
| [organization](variables.tf#L75) | Organization details. | <code title="object({ domain = string })">object({…})</code> | ✓ | |
|
||||
| [prefix](variables.tf#L82) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | <code>string</code> | ✓ | |
|
||||
| [composer_config](variables.tf#L18) | | <code title="object({ node_count = number airflow_version = string env_variables = map(string) })">object({…})</code> | | <code title="{ node_count = 3 airflow_version = "composer-1.17.5-airflow-2.1.4" env_variables = {} }">{…}</code> |
|
||||
| [data_force_destroy](variables.tf#L31) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||
| [groups](variables.tf#L37) | Groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||
| [location_config](variables.tf#L135) | Locations where resources will be deployed. Map to configure region and multiregion specs. | <code title="object({ region = string multi_region = string })">object({…})</code> | | <code title="{ region = "europe-west1" multi_region = "eu" }">{…}</code> |
|
||||
| [project_create](variables.tf#L87) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||
| [project_ids](variables.tf#L96) | Project id, references existing project if `project_create` is null. | <code title="object({ landing = string load = string orchestration = string trasformation = string datalake-l0 = string datalake-l1 = string datalake-l2 = string datalake-playground = string common = string exposure = string })">object({…})</code> | | <code title="{ landing = "lnd" load = "lod" orchestration = "orc" trasformation = "trf" datalake-l0 = "dtl-0" datalake-l1 = "dtl-1" datalake-l2 = "dtl-2" datalake-playground = "dtl-plg" common = "cmn" exposure = "exp" }">{…}</code> |
|
||||
| [project_services](variables.tf#L124) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||
| [billing_account_id](variables.tf#L17) | Billing account id. | <code>string</code> | ✓ | |
|
||||
| [folder_id](variables.tf#L41) | Folder to be used for the networking resources in folders/nnnn format. | <code>string</code> | ✓ | |
|
||||
| [organization_domain](variables.tf#L79) | Organization domain. | <code>string</code> | ✓ | |
|
||||
| [prefix](variables.tf#L84) | Unique prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [composer_config](variables.tf#L22) | | <code title="object({ node_count = number airflow_version = string env_variables = map(string) })">object({…})</code> | | <code title="{ node_count = 3 airflow_version = "composer-1.17.5-airflow-2.1.4" env_variables = {} }">{…}</code> |
|
||||
| [data_force_destroy](variables.tf#L35) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||
| [groups](variables.tf#L46) | Groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||
| [network_config](variables.tf#L56) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string web_server = string }) composer_secondary_ranges = object({ pods = string services = string }) })">object({…})</code> | | <code>null</code> |
|
||||
| [project_services](variables.tf#L89) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||
| [region](variables.tf#L100) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -15,41 +15,13 @@
|
|||
# tfdoc:file:description Core locals.
|
||||
|
||||
locals {
|
||||
_networks = {
|
||||
load = {
|
||||
network_name = coalesce(local._shared_vpc_network, module.lod-vpc[0].name)
|
||||
network = coalesce(var.network_config.network_self_link, module.lod-vpc[0].self_link)
|
||||
subnet = try(var.network_config.subnet_self_links.load, module.lod-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_lod}-subnet"])
|
||||
}
|
||||
orchestration = {
|
||||
network_name = coalesce(local._shared_vpc_network, module.orc-vpc[0].name)
|
||||
network = coalesce(var.network_config.network_self_link, module.orc-vpc[0].self_link)
|
||||
subnet = try(var.network_config.subnet_self_links.orchestration, module.orc-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_orc}-subnet"])
|
||||
}
|
||||
transformation = {
|
||||
network_name = coalesce(local._shared_vpc_network, module.trf-vpc[0].name)
|
||||
network = coalesce(var.network_config.network_self_link, module.trf-vpc[0].self_link)
|
||||
subnet = try(var.network_config.subnet_self_links.transformation, module.trf-vpc[0].subnet_self_links["${var.location_config.region}/${local.prefix_trf}-subnet"])
|
||||
}
|
||||
groups = {
|
||||
for k, v in var.groups : k => "${v}@${var.organization_domain}"
|
||||
}
|
||||
groups_iam = {
|
||||
for k, v in local.groups : k => "group:${v}"
|
||||
}
|
||||
_shared_vpc_network = try(regex("[a-z]([-a-z0-9]*[a-z0-9])?$", var.network_config.network_self_link), null)
|
||||
_shared_vpc_project = try(regex("projects/([a-z0-9-]{6,30})", var.network_config.network_self_link)[0], null)
|
||||
_shared_vpc_service_config = var.network_config.network_self_link != null ? {
|
||||
attach = true
|
||||
host_project = local._shared_vpc_project
|
||||
} : null
|
||||
|
||||
groups = { for k, v in var.groups : k => "${v}@${var.organization.domain}" }
|
||||
groups_iam = { for k, v in local.groups : k => "group:${v}" }
|
||||
service_encryption_keys = var.service_encryption_keys
|
||||
|
||||
# To create KMS keys in the common projet: uncomment assignement below and comment assignement above
|
||||
|
||||
# service_encryption_keys = {
|
||||
# bq = module.sec-kms-1.key_ids.bq
|
||||
# composer = module.sec-kms-2.key_ids.composer
|
||||
# dataflow = module.sec-kms-2.key_ids.dataflow
|
||||
# storage = module.sec-kms-1.key_ids.storage
|
||||
# pubsub = module.sec-kms-0.key_ids.pubsub
|
||||
# }
|
||||
shared_vpc_project = try(var.network_config.host_project, null)
|
||||
use_shared_vpc = var.network_config != null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,25 +17,25 @@
|
|||
output "bigquery-datasets" {
|
||||
description = "BigQuery datasets."
|
||||
value = {
|
||||
lnd-bq-0 = module.lnd-bq-0.dataset_id,
|
||||
dtl-0-bq-0 = module.dtl-0-bq-0.dataset_id,
|
||||
dtl-1-bq-0 = module.dtl-1-bq-0.dataset_id,
|
||||
dtl-2-bq-0 = module.dtl-2-bq-0.dataset_id,
|
||||
dtl-plg-bq-0 = module.dtl-plg-bq-0.dataset_id,
|
||||
land-bq-0 = module.land-bq-0.dataset_id,
|
||||
lake-0-bq-0 = module.lake-0-bq-0.dataset_id,
|
||||
lake-1-bq-0 = module.lake-1-bq-0.dataset_id,
|
||||
lake-2-bq-0 = module.lake-2-bq-0.dataset_id,
|
||||
lake-plg-bq-0 = module.lake-plg-bq-0.dataset_id,
|
||||
}
|
||||
}
|
||||
|
||||
output "gcs-buckets" {
|
||||
description = "GCS buckets."
|
||||
value = {
|
||||
dtl-0-cs-0 = module.dtl-0-cs-0.name,
|
||||
dtl-1-cs-0 = module.dtl-1-cs-0.name,
|
||||
dtl-2-cs-0 = module.dtl-2-cs-0.name,
|
||||
dtl-plg-cs-0 = module.dtl-plg-cs-0.name,
|
||||
lnd-cs-0 = module.lnd-cs-0.name,
|
||||
lod-cs-df = module.lod-cs-df-0.name,
|
||||
orc-cs-0 = module.orc-cs-0.name,
|
||||
trf-cs-df = module.trf-cs-df-0.name,
|
||||
lake-0-cs-0 = module.lake-0-cs-0.name,
|
||||
lake-1-cs-0 = module.lake-1-cs-0.name,
|
||||
lake-2-cs-0 = module.lake-2-cs-0.name,
|
||||
lake-plg-cs-0 = module.lake-plg-cs-0.name,
|
||||
land-cs-0 = module.land-cs-0.name,
|
||||
lod-cs-df = module.load-cs-df-0.name,
|
||||
orch-cs-0 = module.orch-cs-0.name,
|
||||
transf-cs-df = module.transf-cs-df-0.name,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -48,26 +48,26 @@ output "projects" {
|
|||
description = "GCP Projects informations."
|
||||
value = {
|
||||
project_number = {
|
||||
dtl-0-prj = module.dtl-0-prj.number,
|
||||
dtl-1-prj = module.dtl-1-prj.number,
|
||||
dtl-2-prj = module.dtl-2-prj.number,
|
||||
dtl-plg-prj = module.dtl-plg-prj.number,
|
||||
exp-prj = module.exp-prj.number,
|
||||
lnd-prj = module.lnd-prj.number,
|
||||
lod-prj = module.lod-prj.number,
|
||||
orc-prj = module.orc-prj.number,
|
||||
trf-prj = module.trf-prj.number,
|
||||
lake-0 = module.lake-0-project.number,
|
||||
lake-1 = module.lake-1-project.number,
|
||||
lake-2 = module.lake-2-project.number,
|
||||
lake-plg = module.lake-plg-project.number,
|
||||
exposure = module.exp-project.number,
|
||||
landing = module.land-project.number,
|
||||
load = module.load-project.number,
|
||||
orchestration = module.orch-project.number,
|
||||
transformation = module.transf-project.number,
|
||||
}
|
||||
project_id = {
|
||||
dtl-0-prj = module.dtl-0-prj.project_id,
|
||||
dtl-1-prj = module.dtl-1-prj.project_id,
|
||||
dtl-2-prj = module.dtl-2-prj.project_id,
|
||||
dtl-plg-prj = module.dtl-plg-prj.project_id,
|
||||
exp-prj = module.exp-prj.project_id,
|
||||
lnd-prj = module.lnd-prj.project_id,
|
||||
lod-prj = module.lod-prj.project_id,
|
||||
orc-prj = module.orc-prj.project_id,
|
||||
trf-prj = module.trf-prj.project_id,
|
||||
lake-0 = module.lake-0-project.project_id,
|
||||
lake-1 = module.lake-1-project.project_id,
|
||||
lake-2 = module.lake-2-project.project_id,
|
||||
lake-plg = module.lake-plg-project.project_id,
|
||||
exposure = module.exp-project.project_id,
|
||||
landing = module.land-project.project_id,
|
||||
load = module.load-project.project_id,
|
||||
orchestration = module.orch-project.project_id,
|
||||
transformation = module.transf-project.project_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -75,30 +75,30 @@ output "projects" {
|
|||
output "vpc_network" {
|
||||
description = "VPC network."
|
||||
value = {
|
||||
lod-vpc = local._networks.load.network
|
||||
orc-vpc = local._networks.orchestration.network
|
||||
trf-vpc = local._networks.transformation.network
|
||||
load = local.load_vpc
|
||||
orchestration = local.orch_vpc
|
||||
transformation = local.transf_vpc
|
||||
}
|
||||
}
|
||||
|
||||
output "vpc_subnet" {
|
||||
description = "VPC subnetworks."
|
||||
value = {
|
||||
lod-vpc = local._networks.load.subnet
|
||||
orc-vpc = local._networks.orchestration.subnet
|
||||
trf-vpc = local._networks.transformation.subnet
|
||||
load = local.load_subnet
|
||||
orchestration = local.orch_subnet
|
||||
transformation = local.transf_subnet
|
||||
}
|
||||
}
|
||||
|
||||
output "demo_commands" {
|
||||
description = "Demo commands."
|
||||
value = {
|
||||
01 = "gsutil -i ${module.lnd-sa-cs-0.email} cp demo/data/*.csv gs://${module.lnd-cs-0.name}"
|
||||
02 = "gsutil -i ${module.orc-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orc-cs-0.name}"
|
||||
03 = "gsutil -i ${module.orc-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orc-cmp-0.config[0].dag_gcs_prefix}/"
|
||||
04 = "Open ${google_composer_environment.orc-cmp-0.config.0.airflow_uri} and run uploaded DAG."
|
||||
01 = "gsutil -i ${module.land-sa-cs-0.email} cp demo/data/*.csv gs://${module.land-cs-0.name}"
|
||||
02 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/data/*.j* gs://${module.orch-cs-0.name}"
|
||||
03 = "gsutil -i ${module.orch-sa-cmp-0.email} cp demo/*.py ${google_composer_environment.orch-cmp-0.config[0].dag_gcs_prefix}/"
|
||||
04 = "Open ${google_composer_environment.orch-cmp-0.config.0.airflow_uri} and run uploaded DAG."
|
||||
05 = <<EOT
|
||||
bq query --project_id=${module.dtl-2-prj.project_id} --use_legacy_sql=false 'SELECT * FROM `${module.dtl-2-prj.project_id}.${module.dtl-2-bq-0.dataset_id}.customer_purchase` LIMIT 1000'"
|
||||
bq query --project_id=${module.lake-2-project.project_id} --use_legacy_sql=false 'SELECT * FROM `${module.lake-2-project.project_id}.${module.lake-2-bq-0.dataset_id}.customer_purchase` LIMIT 1000'"
|
||||
EOT
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@
|
|||
|
||||
# tfdoc:file:description Terraform Variables.
|
||||
|
||||
variable "billing_account_id" {
|
||||
description = "Billing account id."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "composer_config" {
|
||||
type = object({
|
||||
|
|
@ -34,6 +38,11 @@ variable "data_force_destroy" {
|
|||
default = false
|
||||
}
|
||||
|
||||
variable "folder_id" {
|
||||
description = "Folder to be used for the networking resources in folders/nnnn format."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "groups" {
|
||||
description = "Groups."
|
||||
type = map(string)
|
||||
|
|
@ -47,6 +56,7 @@ variable "groups" {
|
|||
variable "network_config" {
|
||||
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
|
||||
type = object({
|
||||
host_project = string
|
||||
network_self_link = string
|
||||
subnet_self_links = object({
|
||||
load = string
|
||||
|
|
@ -63,62 +73,17 @@ variable "network_config" {
|
|||
services = string
|
||||
})
|
||||
})
|
||||
|
||||
default = {
|
||||
network_self_link = null
|
||||
subnet_self_links = null
|
||||
composer_ip_ranges = null
|
||||
composer_secondary_ranges = null
|
||||
}
|
||||
}
|
||||
|
||||
variable "organization" {
|
||||
description = "Organization details."
|
||||
type = object({
|
||||
domain = string
|
||||
})
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
description = "Unique prefix used for resource names. Not used for projects if 'project_create' is null."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "project_create" {
|
||||
description = "Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
|
||||
type = object({
|
||||
billing_account_id = string
|
||||
parent = string
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "project_ids" {
|
||||
description = "Project id, references existing project if `project_create` is null."
|
||||
type = object({
|
||||
landing = string
|
||||
load = string
|
||||
orchestration = string
|
||||
trasformation = string
|
||||
datalake-l0 = string
|
||||
datalake-l1 = string
|
||||
datalake-l2 = string
|
||||
datalake-playground = string
|
||||
common = string
|
||||
exposure = string
|
||||
})
|
||||
default = {
|
||||
landing = "lnd"
|
||||
load = "lod"
|
||||
orchestration = "orc"
|
||||
trasformation = "trf"
|
||||
datalake-l0 = "dtl-0"
|
||||
datalake-l1 = "dtl-1"
|
||||
datalake-l2 = "dtl-2"
|
||||
datalake-playground = "dtl-plg"
|
||||
common = "cmn"
|
||||
exposure = "exp"
|
||||
}
|
||||
variable "organization_domain" {
|
||||
description = "Organization domain."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
description = "Unique prefix used for resource names."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "project_services" {
|
||||
|
|
@ -132,16 +97,10 @@ variable "project_services" {
|
|||
]
|
||||
}
|
||||
|
||||
variable "location_config" {
|
||||
description = "Locations where resources will be deployed. Map to configure region and multiregion specs."
|
||||
type = object({
|
||||
region = string
|
||||
multi_region = string
|
||||
})
|
||||
default = {
|
||||
region = "europe-west1"
|
||||
multi_region = "eu"
|
||||
}
|
||||
variable "region" {
|
||||
description = "Region used for regional resources."
|
||||
type = string
|
||||
default = "europe-west1"
|
||||
}
|
||||
|
||||
variable "service_encryption_keys" { # service encription key
|
||||
|
|
|
|||
Loading…
Reference in New Issue