zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove unused variables, use regional resources, generate command as output

This commit is contained in:
Julio Castillo 2022-01-13 18:28:38 +01:00
parent 535c9a04a7
commit 90b0d18574
5 changed files with 74 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
examples/data-solutions/gcs-to-bq-with-dataflow/README.md
View File

@ -111,19 +111,19 @@ schema_bq_import.json
You can check data imported into Google BigQuery from the Google Cloud Console UI.
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| billing_account | Billing account id used as default for new projects. | <code>string</code> | ✓ | |
| project_kms_name | Name for the new KMS Project. | <code>string</code> | ✓ | |
| project_service_name | Name for the new Service Project. | <code>string</code> | ✓ | |
| root_node | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | <code>string</code> | ✓ | |
| location | The location where resources will be deployed. | <code>string</code> | | <code>&#34;europe&#34;</code> |
| kms_project_id | Name for the new KMS Project. | <code>string</code> | ✓ | |
| service_project_id | Name for the new Service Project. | <code>string</code> | ✓ | |
| billing_account | Billing account id used as default for new projects. | <code>string</code> | | <code>null</code> |
| project_create | Set to true to create projects, will use existing ones by default. | <code>bool</code> | | <code>false</code> |
| region | The region where resources will be deployed. | <code>string</code> | | <code>&#34;europe-west1&#34;</code> |
| ssh_source_ranges | IP CIDR ranges that will be allowed to connect via SSH to the onprem instance. | <code>list&#40;string&#41;</code> | | <code>&#91;&#34;0.0.0.0&#47;0&#34;&#93;</code> |
| root_node | The resource name of the parent Folder or Organization. Must be of the form folders/folder_id or organizations/org_id. | <code>string</code> | | <code>null</code> |
| vpc_ip_cidr_range | Ip range used in the subnet deployef in the Service Project. | <code>string</code> | | <code>&#34;10.0.0.0&#47;20&#34;</code> |
| vpc_name | Name of the VPC created in the Service Project. | <code>string</code> | | <code>&#34;local&#34;</code> |
| vpc_subnet_name | Name of the subnet created in the Service Project. | <code>string</code> | | <code>&#34;subnet&#34;</code> |
@ -134,8 +134,10 @@ You can check data imported into Google BigQuery from the Google Cloud Console U
|---|---|:---:|
| bq_tables | Bigquery Tables. | |
| buckets | GCS Bucket Cloud KMS crypto keys. | |
| data_ingestion_command | | |
| projects | Project ids. | |
| vm | GCE VM. | |
<!-- END TFDOC -->

67
examples/data-solutions/gcs-to-bq-with-dataflow/main.tf
View File

@ -26,7 +26,7 @@ locals {
module "project-service" {
source = "../../../modules/project"
name = var.project_service_name
name = var.service_project_id
parent = var.root_node
billing_account = var.billing_account
project_create = var.project_create
@ -40,12 +40,13 @@ module "project-service" {
"servicenetworking.googleapis.com",
"storage.googleapis.com",
]
oslogin = true
# TODO(jccb): doesn't work when project_create=false
# oslogin = true
}
module "project-kms" {
source = "../../../modules/project"
name = var.project_kms_name
name = var.kms_project_id
parent = var.root_node
billing_account = var.billing_account
project_create = var.project_create
@ -63,7 +64,7 @@ module "service-account-bq" {
project_id = module.project-service.project_id
name = "bq-test"
iam_project_roles = {
(var.project_service_name) = [
(var.service_project_id) = [
"roles/logging.logWriter",
"roles/monitoring.metricWriter",
"roles/bigquery.admin"
@ -76,7 +77,7 @@ module "service-account-gce" {
project_id = module.project-service.project_id
name = "gce-test"
iam_project_roles = {
(var.project_service_name) = [
(var.service_project_id) = [
"roles/logging.logWriter",
"roles/monitoring.metricWriter",
"roles/dataflow.admin",
@ -92,7 +93,7 @@ module "service-account-df" {
project_id = module.project-service.project_id
name = "df-test"
iam_project_roles = {
(var.project_service_name) = [
(var.service_project_id) = [
"roles/dataflow.worker",
"roles/bigquery.dataOwner",
"roles/bigquery.metadataViewer",
@ -102,14 +103,6 @@ module "service-account-df" {
}
}
# data "google_bigquery_default_service_account" "bq_sa" {
# project = module.project-service.project_id
# }
# data "google_storage_project_service_account" "gcs_account" {
# project = module.project-service.project_id
# }
###############################################################################
# KMS #
###############################################################################
@ -119,39 +112,30 @@ module "kms" {
project_id = module.project-kms.project_id
keyring = {
name = "my-keyring",
location = var.location
location = var.region
}
keys = {
key-df = null
key-gce = null
key-gcs = null
key-bq = null
}
keys = { key-gce = null, key-gcs = null, key-bq = null }
key_iam = {
key-gce = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.project-service.service_accounts.robots.compute}",
"serviceAccount:${module.project-service.service_accounts.robots.compute}"
]
},
key-gcs = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.project-service.service_accounts.robots.storage}",
#"serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"
"serviceAccount:${module.project-service.service_accounts.robots.storage}"
]
},
key-bq = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.project-service.service_accounts.robots.bq}",
#"serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}",
"serviceAccount:${module.project-service.service_accounts.robots.bq}"
]
},
}
}
module "kms-regional" {
source = "../../../modules/kms"
project_id = module.project-kms.project_id
keyring = {
name = "my-keyring-regional",
location = var.region
}
keys = { key-df = null }
key_iam = {
key-df = {
"roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
"serviceAccount:${module.project-service.service_accounts.robots.dataflow}",
@ -161,6 +145,18 @@ module "kms-regional" {
}
}
# module "kms-regional" {
# source = "../../../modules/kms"
# project_id = module.project-kms.project_id
# keyring = {
# name = "my-keyring-regional",
# location = var.region
# }
# keys = { key-df = null }
# key_iam = {
# }
# }
###############################################################################
# Networking #
###############################################################################
@ -198,7 +194,7 @@ module "nat" {
# GCE #
###############################################################################
module "vm_example" {
module "vm" {
source = "../../../modules/compute-vm"
project_id = module.project-service.project_id
zone = "${var.region}-b"
@ -266,7 +262,9 @@ module "kms-gcs" {
project_id = module.project-service.project_id
prefix = module.project-service.project_id
name = each.key
storage_class = "REGIONAL"
iam = each.value.members
location = var.region
encryption_key = module.kms.keys.key-gcs.id
force_destroy = true
}
@ -279,6 +277,7 @@ module "bigquery-dataset" {
source = "../../../modules/bigquery-dataset"
project_id = module.project-service.project_id
id = "bq_dataset"
location = var.region
access = {
reader-group = { role = "READER", type = "user" }
owner = { role = "OWNER", type = "user" }

24
examples/data-solutions/gcs-to-bq-with-dataflow/outputs.tf
View File

@ -25,6 +25,26 @@ output "buckets" {
}
}
output "data_ingestion_command" {
value = <<-EOF
python data_ingestion.py \
--runner=DataflowRunner \
--max_num_workers=10 \
--autoscaling_algorithm=THROUGHPUT_BASED \
--region=${var.region} \
--staging_location=${module.kms-gcs["df-tmplocation"].url} \
--temp_location=${module.kms-gcs["df-tmplocation"].url}/ \
--project=${var.service_project_id} \
--input=${module.kms-gcs["data"].url}/### FILE NAME ###.csv \
--output=${module.bigquery-dataset.dataset_id}.${module.bigquery-dataset.table_ids.df_import} \
--service_account_email=${module.service-account-df.email} \
--network=${var.vpc_name} \
--subnetwork=${var.vpc_subnet_name} \
--dataflow_kms_key=${module.kms.key_ids.key-df} \
--no_use_public_ips
EOF
}
output "projects" {
description = "Project ids."
value = {
@ -36,7 +56,7 @@ output "projects" {
output "vm" {
description = "GCE VM."
value = {
name = module.vm_example.instance.name
address = module.vm_example.internal_ip
name = module.vm.instance.name
address = module.vm.internal_ip
}
}

21
examples/data-solutions/gcs-to-bq-with-dataflow/variables.tf
View File

@ -19,24 +19,18 @@ variable "billing_account" {
default = null
}
variable "location" {
description = "The location where resources will be deployed."
type = string
default = "europe"
}
variable "project_create" {
description = "Set to true to create projects, will use existing ones by default."
type = bool
default = false
}
variable "project_kms_name" {
variable "kms_project_id" {
description = "Name for the new KMS Project."
type = string
}
variable "project_service_name" {
variable "service_project_id" {
description = "Name for the new Service Project."
type = string
}
@ -53,11 +47,12 @@ variable "root_node" {
default = null
}
variable "ssh_source_ranges" {
description = "IP CIDR ranges that will be allowed to connect via SSH to the onprem instance."
type = list(string)
default = ["0.0.0.0/0"]
}
# FIXME(jccb): this is not used
# variable "ssh_source_ranges" {
# description = "IP CIDR ranges that will be allowed to connect via SSH to the onprem instance."
# type = list(string)
# default = ["0.0.0.0/0"]
# }
variable "vpc_ip_cidr_range" {
description = "Ip range used in the subnet deployef in the Service Project."

4
modules/project/outputs.tf
View File

@ -70,7 +70,9 @@ output "service_accounts" {
depends_on = [
google_project_service.project_services,
google_kms_crypto_key_iam_member.crypto_key,
google_project_service_identity.jit_si
google_project_service_identity.jit_si,
data.google_bigquery_default_service_account.bq_sa,
data.google_storage_project_service_account.gcs_sa
]
}