zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split resources in separate TF files

This commit is contained in:
Lorenzo Caggioni 2022-01-14 08:45:14 +01:00
parent d8de100ff5
commit 15e573a1bf
4 changed files with 164 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
data-solutions/gcs-to-bq-with-least-privileges/datastorage.tf Normal file
View File

@ -0,0 +1,39 @@
###############################################################################
# GCS #
###############################################################################
module "gcs-01" {
source = "../../modules/gcs"
for_each = toset(["data-landing", "df-tmplocation"])
project_id = module.project-service.project_id
prefix = module.project-service.project_id
name = each.key
force_destroy = true
}
###############################################################################
# BQ #
###############################################################################
module "bigquery-dataset" {
source = "../../modules/bigquery-dataset"
project_id = module.project-service.project_id
id = "datalake"
# Define Tables in Terraform for the porpuse of the example.
# Probably in a production environment you would handle Tables creation in a
# separate Terraform State or using a different tool/pipeline (for example: Dataform).
tables = {
person = {
friendly_name = "Person. Dataflow import."
labels = {}
options = null
partitioning = {
field = null
range = null # use start/end/interval for range
time = null
}
schema = file("${path.module}/person.json")
deletion_protection = false
}
}
}

147
data-solutions/gcs-to-bq-with-least-privileges/main.tf
View File

@ -25,7 +25,7 @@ locals {
}
###############################################################################
# Projects - Centralized #
# Projects #
###############################################################################
module "project-service" {
@ -98,148 +98,3 @@ module "project-service" {
}
oslogin = true
}
###############################################################################
# Project Service Accounts #
###############################################################################
module "service-account-bq" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "bq-datalake"
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
module "service-account-landing" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "gcs-landing"
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
module "service-account-orch" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "orchestrator"
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
module "service-account-df" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "df-loading"
iam_project_roles = {
(var.project_name) = [
"roles/dataflow.worker",
"roles/bigquery.dataOwner",
"roles/bigquery.metadataViewer",
"roles/storage.objectViewer",
"roles/bigquery.jobUser"
]
}
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
),
"roles/iam.serviceAccountUser" = concat(
[module.service-account-orch.iam_email],
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
###############################################################################
# Networking #
###############################################################################
module "vpc" {
source = "../../modules/net-vpc"
project_id = module.project-service.project_id
name = var.vpc_name
subnets = [
{
ip_cidr_range = var.vpc_ip_cidr_range
name = var.vpc_subnet_name
region = var.region
secondary_ip_range = {}
}
]
}
module "vpc-firewall" {
source = "../../modules/net-vpc-firewall"
project_id = module.project-service.project_id
network = module.vpc.name
admin_ranges = [var.vpc_ip_cidr_range]
}
module "nat" {
source = "../../modules/net-cloudnat"
project_id = module.project-service.project_id
region = var.region
name = "default"
router_network = module.vpc.name
}
###############################################################################
# GCS #
###############################################################################
module "gcs-01" {
source = "../../modules/gcs"
for_each = toset(["data-landing", "df-tmplocation"])
project_id = module.project-service.project_id
prefix = module.project-service.project_id
name = each.key
force_destroy = true
}
# module "gcs-02" {
# source = "../../modules/gcs-demo"
# project_id = module.project-service.project_id
# prefix = module.project-service.project_id
# name = "test-region"
# location = "europe-west1"
# storage_class = "REGIONAL"
# force_destroy = true
# }
###############################################################################
# BQ #
###############################################################################
module "bigquery-dataset" {
source = "../../modules/bigquery-dataset"
project_id = module.project-service.project_id
id = "datalake"
tables = {
person = {
friendly_name = "Person. Dataflow import."
labels = {}
options = null
partitioning = {
field = null
range = null # use start/end/interval for range
time = null
}
schema = file("${path.module}/person.json")
deletion_protection = false
}
}
}

78
data-solutions/gcs-to-bq-with-least-privileges/serviceaccounts.tf Normal file
View File

@ -0,0 +1,78 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Service Accounts #
###############################################################################
module "service-account-bq" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "bq-datalake"
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
module "service-account-landing" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "gcs-landing"
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
module "service-account-orch" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "orchestrator"
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}
module "service-account-df" {
source = "../../modules/iam-service-account"
project_id = module.project-service.project_id
name = "df-loading"
iam_project_roles = {
(var.project_name) = [
"roles/dataflow.worker",
"roles/bigquery.dataOwner",
"roles/bigquery.metadataViewer",
"roles/storage.objectViewer",
"roles/bigquery.jobUser"
]
}
iam = {
"roles/iam.serviceAccountTokenCreator" = concat(
local.data_eng_users_iam,
local.data_eng_groups_iam
),
"roles/iam.serviceAccountUser" = concat(
[module.service-account-orch.iam_email],
local.data_eng_users_iam,
local.data_eng_groups_iam
)
}
}

46
data-solutions/gcs-to-bq-with-least-privileges/vpc.tf Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Networking #
###############################################################################
module "vpc" {
source = "../../modules/net-vpc"
project_id = module.project-service.project_id
name = var.vpc_name
subnets = [
{
ip_cidr_range = var.vpc_ip_cidr_range
name = var.vpc_subnet_name
region = var.region
secondary_ip_range = {}
}
]
}
module "vpc-firewall" {
source = "../../modules/net-vpc-firewall"
project_id = module.project-service.project_id
network = module.vpc.name
admin_ranges = [var.vpc_ip_cidr_range]
}
module "nat" {
source = "../../modules/net-cloudnat"
project_id = module.project-service.project_id
region = var.region
name = "default"
router_network = module.vpc.name
}