Merge pull request #601 from GoogleCloudPlatform/lcaggio/gcs2bq_shared_vpc
[data-solutions/gcs-to-bq-with-least-privileges] Add support for shared VPC
This commit is contained in:
commit
8a158054bb
|
@ -74,6 +74,16 @@ $ terraform apply
|
|||
|
||||
You should see the output of the Terraform script with resources created and some command pre-created for you to run the example following steps below.
|
||||
|
||||
### Virtual Private Cloud (VPC) design
|
||||
|
||||
As is often the case in real-world configurations, this example accepts as input an existing [Shared-VPC](https://cloud.google.com/vpc/docs/shared-vpc) via the `network_config` variable.
|
||||
|
||||
If the `network_config` variable is not provided, one VPC will be created in each project that supports network resources (load, transformation and orchestration).
|
||||
|
||||
When `network_config` variable is configured, the identity running the Terraform script need to have the following roles:
|
||||
- `roles/compute.xpnAdmin` on the host project folder or org
|
||||
- `roles/resourcemanager.projectIamAdmin` on the host project, either with no conditions or with a condition allowing [delegated role grants](https://medium.com/google-cloud/managing-gcp-service-usage-through-delegated-role-grants-a843610f2226#:~:text=Delegated%20role%20grants%20is%20a,setIamPolicy%20permission%20on%20a%20resource.) for `roles/compute.networkUser`, `roles/composer.sharedVpcAgent`, `roles/container.hostServiceAgentUser`
|
||||
|
||||
## Test your environment with Cloud Dataflow
|
||||
|
||||
We assume all those steps are run using a user listed on `data_eng_principals`. You can authenticate as the user using the following command:
|
||||
|
@ -131,13 +141,14 @@ bq query --use_legacy_sql=false 'SELECT * FROM `PROJECT.datalake.person` LIMIT 1
|
|||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [prefix](variables.tf#L26) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L40) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
||||
| [prefix](variables.tf#L36) | Unique prefix used for resource names. Not used for project if 'project_create' is null. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L50) | Project id, references existing project if `project_create` is null. | <code>string</code> | ✓ | |
|
||||
| [cmek_encryption](variables.tf#L15) | Flag to enable CMEK on GCP resources created. | <code>bool</code> | | <code>false</code> |
|
||||
| [data_eng_principals](variables.tf#L21) | Groups with Service Account Token creator role on service accounts in IAM format, eg 'group:group@domain.com'. | <code>list(string)</code> | | <code>[]</code> |
|
||||
| [project_create](variables.tf#L31) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L45) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [vpc_subnet_range](variables.tf#L51) | Ip range used for the VPC subnet created for the example. | <code>string</code> | | <code>"10.0.0.0/20"</code> |
|
||||
| [network_config](variables.tf#L27) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string subnet_self_link = string })">object({…})</code> | | <code>null</code> |
|
||||
| [project_create](variables.tf#L41) | Provide values if project creation is needed, uses existing project if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = string parent = string })">object({…})</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L55) | The region where resources will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [vpc_subnet_range](variables.tf#L61) | Ip range used for the VPC subnet created for the example. | <code>string</code> | | <code>"10.0.0.0/20"</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
@ -76,6 +76,29 @@ locals {
|
|||
"serviceAccount:${module.project.service_accounts.robots.dataflow}"
|
||||
]
|
||||
}
|
||||
network_subnet_selflink = try(
|
||||
module.vpc[0].subnets["${var.region}/subnet"].self_link,
|
||||
var.network_config.subnet_self_link
|
||||
)
|
||||
shared_vpc_bindings = {
|
||||
"roles/compute.networkUser" = [
|
||||
"robot-df", "sa-df-worker"
|
||||
]
|
||||
}
|
||||
# reassemble in a format suitable for for_each
|
||||
shared_vpc_bindings_map = {
|
||||
for binding in flatten([
|
||||
for role, members in local.shared_vpc_bindings : [
|
||||
for member in members : { role = role, member = member }
|
||||
]
|
||||
]) : "${binding.role}-${binding.member}" => binding
|
||||
}
|
||||
shared_vpc_project = try(var.network_config.host_project, null)
|
||||
shared_vpc_role_members = {
|
||||
robot-df = "serviceAccount:${module.project.service_accounts.robots.dataflow}"
|
||||
sa-df-worker = module.service-account-df.iam_email
|
||||
}
|
||||
use_shared_vpc = var.network_config != null
|
||||
}
|
||||
|
||||
module "project" {
|
||||
|
@ -100,7 +123,19 @@ module "project" {
|
|||
# additive IAM bindings avoid disrupting bindings in existing project
|
||||
iam = var.project_create != null ? local.iam : {}
|
||||
iam_additive = var.project_create == null ? local.iam : {}
|
||||
shared_vpc_service_config = local.shared_vpc_project == null ? null : {
|
||||
attach = true
|
||||
host_project = local.shared_vpc_project
|
||||
service_identity_iam = {}
|
||||
}
|
||||
service_config = {
|
||||
disable_on_destroy = false, disable_dependent_services = false
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_project_iam_member" "shared_vpc" {
|
||||
for_each = local.use_shared_vpc ? local.shared_vpc_bindings_map : {}
|
||||
project = var.network_config.host_project
|
||||
role = each.value.role
|
||||
member = lookup(local.shared_vpc_role_members, each.value.member)
|
||||
}
|
||||
|
|
|
@ -51,7 +51,7 @@ output "command_02_dataflow" {
|
|||
sa_orch_email = module.service-account-orch.email
|
||||
project_id = module.project.project_id
|
||||
region = var.region
|
||||
subnet = module.vpc.subnets["${var.region}/subnet"].self_link
|
||||
subnet = local.network_subnet_selflink
|
||||
gcs_df_stg = format("%s/%s", module.gcs-df-tmp.url, "stg")
|
||||
sa_df_email = module.service-account-df.email
|
||||
cmek_encryption = var.cmek_encryption
|
||||
|
|
|
@ -23,6 +23,16 @@ variable "data_eng_principals" {
|
|||
type = list(string)
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "network_config" {
|
||||
description = "Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values."
|
||||
type = object({
|
||||
host_project = string
|
||||
subnet_self_link = string
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
description = "Unique prefix used for resource names. Not used for project if 'project_create' is null."
|
||||
type = string
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
module "vpc" {
|
||||
source = "../../../modules/net-vpc"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
name = "${var.prefix}-vpc"
|
||||
subnets = [
|
||||
|
@ -28,15 +29,17 @@ module "vpc" {
|
|||
|
||||
module "vpc-firewall" {
|
||||
source = "../../../modules/net-vpc-firewall"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
network = module.vpc.name
|
||||
network = module.vpc[0].name
|
||||
admin_ranges = [var.vpc_subnet_range]
|
||||
}
|
||||
|
||||
module "nat" {
|
||||
source = "../../../modules/net-cloudnat"
|
||||
count = local.use_shared_vpc ? 0 : 1
|
||||
project_id = module.project.project_id
|
||||
region = var.region
|
||||
name = "${var.prefix}-default"
|
||||
router_network = module.vpc.name
|
||||
router_network = module.vpc[0].name
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue