From 15e573a1bf3502344dd77b869557e0311254c71d Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Fri, 14 Jan 2022 08:45:14 +0100 Subject: [PATCH] Split resources in separate TF files --- .../datastorage.tf | 39 +++++ .../gcs-to-bq-with-least-privileges/main.tf | 147 +----------------- .../serviceaccounts.tf | 78 ++++++++++ .../gcs-to-bq-with-least-privileges/vpc.tf | 46 ++++++ 4 files changed, 164 insertions(+), 146 deletions(-) create mode 100644 data-solutions/gcs-to-bq-with-least-privileges/datastorage.tf create mode 100644 data-solutions/gcs-to-bq-with-least-privileges/serviceaccounts.tf create mode 100644 data-solutions/gcs-to-bq-with-least-privileges/vpc.tf diff --git a/data-solutions/gcs-to-bq-with-least-privileges/datastorage.tf b/data-solutions/gcs-to-bq-with-least-privileges/datastorage.tf new file mode 100644 index 00000000..c3b6e2d4 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-least-privileges/datastorage.tf @@ -0,0 +1,39 @@ +############################################################################### +# GCS # +############################################################################### + +module "gcs-01" { + source = "../../modules/gcs" + for_each = toset(["data-landing", "df-tmplocation"]) + project_id = module.project-service.project_id + prefix = module.project-service.project_id + name = each.key + force_destroy = true +} + +############################################################################### +# BQ # +############################################################################### + +module "bigquery-dataset" { + source = "../../modules/bigquery-dataset" + project_id = module.project-service.project_id + id = "datalake" + # Define Tables in Terraform for the porpuse of the example. + # Probably in a production environment you would handle Tables creation in a + # separate Terraform State or using a different tool/pipeline (for example: Dataform). + tables = { + person = { + friendly_name = "Person. Dataflow import." + labels = {} + options = null + partitioning = { + field = null + range = null # use start/end/interval for range + time = null + } + schema = file("${path.module}/person.json") + deletion_protection = false + } + } +} diff --git a/data-solutions/gcs-to-bq-with-least-privileges/main.tf b/data-solutions/gcs-to-bq-with-least-privileges/main.tf index f49d02ac..b80fed7f 100644 --- a/data-solutions/gcs-to-bq-with-least-privileges/main.tf +++ b/data-solutions/gcs-to-bq-with-least-privileges/main.tf @@ -25,7 +25,7 @@ locals { } ############################################################################### -# Projects - Centralized # +# Projects # ############################################################################### module "project-service" { @@ -98,148 +98,3 @@ module "project-service" { } oslogin = true } - -############################################################################### -# Project Service Accounts # -############################################################################### - -module "service-account-bq" { - source = "../../modules/iam-service-account" - project_id = module.project-service.project_id - name = "bq-datalake" - iam = { - "roles/iam.serviceAccountTokenCreator" = concat( - local.data_eng_users_iam, - local.data_eng_groups_iam - ) - } -} -module "service-account-landing" { - source = "../../modules/iam-service-account" - project_id = module.project-service.project_id - name = "gcs-landing" - iam = { - "roles/iam.serviceAccountTokenCreator" = concat( - local.data_eng_users_iam, - local.data_eng_groups_iam - ) - } -} - -module "service-account-orch" { - source = "../../modules/iam-service-account" - project_id = module.project-service.project_id - name = "orchestrator" - iam = { - "roles/iam.serviceAccountTokenCreator" = concat( - local.data_eng_users_iam, - local.data_eng_groups_iam - ) - } -} - -module "service-account-df" { - source = "../../modules/iam-service-account" - project_id = module.project-service.project_id - name = "df-loading" - iam_project_roles = { - (var.project_name) = [ - "roles/dataflow.worker", - "roles/bigquery.dataOwner", - "roles/bigquery.metadataViewer", - "roles/storage.objectViewer", - "roles/bigquery.jobUser" - ] - } - iam = { - "roles/iam.serviceAccountTokenCreator" = concat( - local.data_eng_users_iam, - local.data_eng_groups_iam - ), - "roles/iam.serviceAccountUser" = concat( - [module.service-account-orch.iam_email], - local.data_eng_users_iam, - local.data_eng_groups_iam - ) - } -} - -############################################################################### -# Networking # -############################################################################### - -module "vpc" { - source = "../../modules/net-vpc" - project_id = module.project-service.project_id - name = var.vpc_name - subnets = [ - { - ip_cidr_range = var.vpc_ip_cidr_range - name = var.vpc_subnet_name - region = var.region - secondary_ip_range = {} - } - ] -} - -module "vpc-firewall" { - source = "../../modules/net-vpc-firewall" - project_id = module.project-service.project_id - network = module.vpc.name - admin_ranges = [var.vpc_ip_cidr_range] -} - -module "nat" { - source = "../../modules/net-cloudnat" - project_id = module.project-service.project_id - region = var.region - name = "default" - router_network = module.vpc.name -} - -############################################################################### -# GCS # -############################################################################### - -module "gcs-01" { - source = "../../modules/gcs" - for_each = toset(["data-landing", "df-tmplocation"]) - project_id = module.project-service.project_id - prefix = module.project-service.project_id - name = each.key - force_destroy = true -} - -# module "gcs-02" { -# source = "../../modules/gcs-demo" -# project_id = module.project-service.project_id -# prefix = module.project-service.project_id -# name = "test-region" -# location = "europe-west1" -# storage_class = "REGIONAL" -# force_destroy = true -# } - -############################################################################### -# BQ # -############################################################################### - -module "bigquery-dataset" { - source = "../../modules/bigquery-dataset" - project_id = module.project-service.project_id - id = "datalake" - tables = { - person = { - friendly_name = "Person. Dataflow import." - labels = {} - options = null - partitioning = { - field = null - range = null # use start/end/interval for range - time = null - } - schema = file("${path.module}/person.json") - deletion_protection = false - } - } -} diff --git a/data-solutions/gcs-to-bq-with-least-privileges/serviceaccounts.tf b/data-solutions/gcs-to-bq-with-least-privileges/serviceaccounts.tf new file mode 100644 index 00000000..627844d9 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-least-privileges/serviceaccounts.tf @@ -0,0 +1,78 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Service Accounts # +############################################################################### + +module "service-account-bq" { + source = "../../modules/iam-service-account" + project_id = module.project-service.project_id + name = "bq-datalake" + iam = { + "roles/iam.serviceAccountTokenCreator" = concat( + local.data_eng_users_iam, + local.data_eng_groups_iam + ) + } +} +module "service-account-landing" { + source = "../../modules/iam-service-account" + project_id = module.project-service.project_id + name = "gcs-landing" + iam = { + "roles/iam.serviceAccountTokenCreator" = concat( + local.data_eng_users_iam, + local.data_eng_groups_iam + ) + } +} + +module "service-account-orch" { + source = "../../modules/iam-service-account" + project_id = module.project-service.project_id + name = "orchestrator" + iam = { + "roles/iam.serviceAccountTokenCreator" = concat( + local.data_eng_users_iam, + local.data_eng_groups_iam + ) + } +} + +module "service-account-df" { + source = "../../modules/iam-service-account" + project_id = module.project-service.project_id + name = "df-loading" + iam_project_roles = { + (var.project_name) = [ + "roles/dataflow.worker", + "roles/bigquery.dataOwner", + "roles/bigquery.metadataViewer", + "roles/storage.objectViewer", + "roles/bigquery.jobUser" + ] + } + iam = { + "roles/iam.serviceAccountTokenCreator" = concat( + local.data_eng_users_iam, + local.data_eng_groups_iam + ), + "roles/iam.serviceAccountUser" = concat( + [module.service-account-orch.iam_email], + local.data_eng_users_iam, + local.data_eng_groups_iam + ) + } +} diff --git a/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf b/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf new file mode 100644 index 00000000..e13ea6a6 --- /dev/null +++ b/data-solutions/gcs-to-bq-with-least-privileges/vpc.tf @@ -0,0 +1,46 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Networking # +############################################################################### + +module "vpc" { + source = "../../modules/net-vpc" + project_id = module.project-service.project_id + name = var.vpc_name + subnets = [ + { + ip_cidr_range = var.vpc_ip_cidr_range + name = var.vpc_subnet_name + region = var.region + secondary_ip_range = {} + } + ] +} + +module "vpc-firewall" { + source = "../../modules/net-vpc-firewall" + project_id = module.project-service.project_id + network = module.vpc.name + admin_ranges = [var.vpc_ip_cidr_range] +} + +module "nat" { + source = "../../modules/net-cloudnat" + project_id = module.project-service.project_id + region = var.region + name = "default" + router_network = module.vpc.name +}