Minimal Data Platform - Make components optional (#1380)
make some components optional: Composer and Data Proc history server.
This commit is contained in:
parent
6a89d71e96
commit
40656a23de
|
@ -18,7 +18,7 @@ locals {
|
||||||
iam_lnd = {
|
iam_lnd = {
|
||||||
"roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
|
"roles/storage.objectCreator" = [module.land-sa-cs-0.iam_email]
|
||||||
"roles/storage.objectViewer" = [module.processing-sa-cmp-0.iam_email]
|
"roles/storage.objectViewer" = [module.processing-sa-cmp-0.iam_email]
|
||||||
"roles/storage.objectAdmin" = [module.processing-sa-dp-0.iam_email]
|
"roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,10 +25,10 @@ locals {
|
||||||
GCP_REGION = var.region
|
GCP_REGION = var.region
|
||||||
LAND_PRJ = module.land-project.project_id
|
LAND_PRJ = module.land-project.project_id
|
||||||
LAND_GCS = module.land-cs-0.name
|
LAND_GCS = module.land-cs-0.name
|
||||||
PHS_CLUSTER_NAME = module.processing-dp-historyserver.name
|
PHS_CLUSTER_NAME = try(module.processing-dp-historyserver[0].name, null)
|
||||||
PROCESSING_GCS = module.processing-cs-0.name
|
PROCESSING_GCS = module.processing-cs-0.name
|
||||||
PROCESSING_PRJ = module.processing-project.project_id
|
PROCESSING_PRJ = module.processing-project.project_id
|
||||||
PROCESSING_SA_DP = module.processing-sa-dp-0.email
|
PROCESSING_SA = module.processing-sa-0.email
|
||||||
PROCESSING_SUBNET = local.processing_subnet
|
PROCESSING_SUBNET = local.processing_subnet
|
||||||
PROCESSING_VPC = local.processing_vpc
|
PROCESSING_VPC = local.processing_vpc
|
||||||
}
|
}
|
||||||
|
@ -47,7 +47,7 @@ module "processing-sa-cmp-0" {
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_composer_environment" "processing-cmp-0" {
|
resource "google_composer_environment" "processing-cmp-0" {
|
||||||
count = var.composer_config.disable_deployment == true ? 0 : 1
|
count = var.enable_services.composer == true ? 1 : 0
|
||||||
project = module.processing-project.project_id
|
project = module.processing-project.project_id
|
||||||
name = "${var.prefix}-prc-cmp-0"
|
name = "${var.prefix}-prc-cmp-0"
|
||||||
region = var.region
|
region = var.region
|
||||||
|
|
|
@ -14,7 +14,8 @@
|
||||||
|
|
||||||
# tfdoc:file:description Cloud Dataproc resources.
|
# tfdoc:file:description Cloud Dataproc resources.
|
||||||
|
|
||||||
module "processing-cs-dp-history" {
|
module "processing-dp-history" {
|
||||||
|
count = var.enable_services.dataproc_history_server == true ? 1 : 0
|
||||||
source = "../../../modules/gcs"
|
source = "../../../modules/gcs"
|
||||||
project_id = module.processing-project.project_id
|
project_id = module.processing-project.project_id
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
|
@ -24,12 +25,12 @@ module "processing-cs-dp-history" {
|
||||||
encryption_key = var.service_encryption_keys.storage
|
encryption_key = var.service_encryption_keys.storage
|
||||||
}
|
}
|
||||||
|
|
||||||
module "processing-sa-dp-0" {
|
module "processing-sa-0" {
|
||||||
source = "../../../modules/iam-service-account"
|
source = "../../../modules/iam-service-account"
|
||||||
project_id = module.processing-project.project_id
|
project_id = module.processing-project.project_id
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
name = "prc-dp-0"
|
name = "prc-0"
|
||||||
display_name = "Dataproc service account"
|
display_name = "Processing service account"
|
||||||
iam = {
|
iam = {
|
||||||
"roles/iam.serviceAccountTokenCreator" = [
|
"roles/iam.serviceAccountTokenCreator" = [
|
||||||
local.groups_iam.data-engineers,
|
local.groups_iam.data-engineers,
|
||||||
|
@ -41,7 +42,7 @@ module "processing-sa-dp-0" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module "processing-dp-staging-0" {
|
module "processing-staging-0" {
|
||||||
source = "../../../modules/gcs"
|
source = "../../../modules/gcs"
|
||||||
project_id = module.processing-project.project_id
|
project_id = module.processing-project.project_id
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
|
@ -51,7 +52,7 @@ module "processing-dp-staging-0" {
|
||||||
encryption_key = var.service_encryption_keys.storage
|
encryption_key = var.service_encryption_keys.storage
|
||||||
}
|
}
|
||||||
|
|
||||||
module "processing-dp-temp-0" {
|
module "processing-temp-0" {
|
||||||
source = "../../../modules/gcs"
|
source = "../../../modules/gcs"
|
||||||
project_id = module.processing-project.project_id
|
project_id = module.processing-project.project_id
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
|
@ -61,7 +62,7 @@ module "processing-dp-temp-0" {
|
||||||
encryption_key = var.service_encryption_keys.storage
|
encryption_key = var.service_encryption_keys.storage
|
||||||
}
|
}
|
||||||
|
|
||||||
module "processing-dp-log-0" {
|
module "processing-log-0" {
|
||||||
source = "../../../modules/gcs"
|
source = "../../../modules/gcs"
|
||||||
project_id = module.processing-project.project_id
|
project_id = module.processing-project.project_id
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
|
@ -72,19 +73,20 @@ module "processing-dp-log-0" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "processing-dp-historyserver" {
|
module "processing-dp-historyserver" {
|
||||||
|
count = var.enable_services.dataproc_history_server == true ? 1 : 0
|
||||||
source = "../../../modules/dataproc"
|
source = "../../../modules/dataproc"
|
||||||
project_id = module.processing-project.project_id
|
project_id = module.processing-project.project_id
|
||||||
name = "hystory-server"
|
name = "history-server"
|
||||||
prefix = var.prefix
|
prefix = var.prefix
|
||||||
region = var.region
|
region = var.region
|
||||||
dataproc_config = {
|
dataproc_config = {
|
||||||
cluster_config = {
|
cluster_config = {
|
||||||
staging_bucket = module.processing-dp-staging-0.name
|
staging_bucket = module.processing-staging-0.name
|
||||||
temp_bucket = module.processing-dp-temp-0.name
|
temp_bucket = module.processing-temp-0.name
|
||||||
gce_cluster_config = {
|
gce_cluster_config = {
|
||||||
subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link
|
subnetwork = module.processing-vpc[0].subnets["${var.region}/${var.prefix}-processing"].self_link
|
||||||
zone = "${var.region}-b"
|
zone = "${var.region}-b"
|
||||||
service_account = module.processing-sa-dp-0.email
|
service_account = module.processing-sa-0.email
|
||||||
service_account_scopes = ["cloud-platform"]
|
service_account_scopes = ["cloud-platform"]
|
||||||
internal_ip_only = true
|
internal_ip_only = true
|
||||||
}
|
}
|
||||||
|
@ -99,10 +101,10 @@ module "processing-dp-historyserver" {
|
||||||
"dataproc:dataproc.allow.zero.workers" = "true"
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
||||||
"dataproc:job.history.to-gcs.enabled" = "true"
|
"dataproc:job.history.to-gcs.enabled" = "true"
|
||||||
"spark:spark.history.fs.logDirectory" = (
|
"spark:spark.history.fs.logDirectory" = (
|
||||||
"gs://${module.processing-dp-staging-0.name}/*/spark-job-history"
|
"gs://${module.processing-staging-0.name}/*/spark-job-history"
|
||||||
)
|
)
|
||||||
"spark:spark.eventLog.dir" = (
|
"spark:spark.eventLog.dir" = (
|
||||||
"gs://${module.processing-dp-staging-0.name}/*/spark-job-history"
|
"gs://${module.processing-staging-0.name}/*/spark-job-history"
|
||||||
)
|
)
|
||||||
"spark:spark.history.custom.executor.log.url.applyIncompleteApplication" = "false"
|
"spark:spark.history.custom.executor.log.url.applyIncompleteApplication" = "false"
|
||||||
"spark:spark.history.custom.executor.log.url" = (
|
"spark:spark.history.custom.executor.log.url" = (
|
||||||
|
|
|
@ -28,7 +28,7 @@ locals {
|
||||||
module.processing-sa-cmp-0.iam_email
|
module.processing-sa-cmp-0.iam_email
|
||||||
]
|
]
|
||||||
"roles/dataproc.worker" = [
|
"roles/dataproc.worker" = [
|
||||||
module.processing-sa-dp-0.iam_email
|
module.processing-sa-0.iam_email
|
||||||
]
|
]
|
||||||
"roles/iam.serviceAccountUser" = [
|
"roles/iam.serviceAccountUser" = [
|
||||||
module.processing-sa-cmp-0.iam_email, local.groups_iam.data-engineers
|
module.processing-sa-cmp-0.iam_email, local.groups_iam.data-engineers
|
||||||
|
|
|
@ -16,13 +16,13 @@
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
cur_iam = {
|
cur_iam = {
|
||||||
"roles/bigquery.dataOwner" = [module.processing-sa-dp-0.iam_email]
|
"roles/bigquery.dataOwner" = [module.processing-sa-0.iam_email]
|
||||||
"roles/bigquery.dataViewer" = [
|
"roles/bigquery.dataViewer" = [
|
||||||
local.groups_iam.data-analysts,
|
local.groups_iam.data-analysts,
|
||||||
local.groups_iam.data-engineers
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/bigquery.jobUser" = [
|
"roles/bigquery.jobUser" = [
|
||||||
module.processing-sa-dp-0.iam_email,
|
module.processing-sa-0.iam_email,
|
||||||
local.groups_iam.data-analysts,
|
local.groups_iam.data-analysts,
|
||||||
local.groups_iam.data-engineers
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
|
@ -35,7 +35,7 @@ locals {
|
||||||
"roles/storage.objectViewer" = [
|
"roles/storage.objectViewer" = [
|
||||||
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
local.groups_iam.data-analysts, local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/storage.objectAdmin" = [module.processing-sa-dp-0.iam_email]
|
"roles/storage.objectAdmin" = [module.processing-sa-0.iam_email]
|
||||||
}
|
}
|
||||||
cur_services = [
|
cur_services = [
|
||||||
"iam.googleapis.com",
|
"iam.googleapis.com",
|
||||||
|
|
|
@ -20,16 +20,16 @@ locals {
|
||||||
"roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers]
|
"roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers]
|
||||||
"roles/dlp.reader" = [local.groups_iam.data-engineers]
|
"roles/dlp.reader" = [local.groups_iam.data-engineers]
|
||||||
"roles/dlp.user" = [
|
"roles/dlp.user" = [
|
||||||
module.processing-sa-dp-0.iam_email,
|
module.processing-sa-0.iam_email,
|
||||||
local.groups_iam.data-engineers
|
local.groups_iam.data-engineers
|
||||||
]
|
]
|
||||||
"roles/datacatalog.admin" = [local.groups_iam.data-security]
|
"roles/datacatalog.admin" = [local.groups_iam.data-security]
|
||||||
"roles/datacatalog.viewer" = [
|
"roles/datacatalog.viewer" = [
|
||||||
module.processing-sa-dp-0.iam_email,
|
module.processing-sa-0.iam_email,
|
||||||
local.groups_iam.data-analysts
|
local.groups_iam.data-analysts
|
||||||
]
|
]
|
||||||
"roles/datacatalog.categoryFineGrainedReader" = [
|
"roles/datacatalog.categoryFineGrainedReader" = [
|
||||||
module.processing-sa-dp-0.iam_email
|
module.processing-sa-0.iam_email
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -230,7 +230,7 @@ network_config = {
|
||||||
host_project = "PROJECT_ID"
|
host_project = "PROJECT_ID"
|
||||||
network_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/global/networks/NAME"
|
network_self_link = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/global/networks/NAME"
|
||||||
subnet_self_links = {
|
subnet_self_links = {
|
||||||
processing_dataproc = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
processing_transformation = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
||||||
processing_composer = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
processing_composer = "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/NAME"
|
||||||
}
|
}
|
||||||
composer_ip_ranges = {
|
composer_ip_ranges = {
|
||||||
|
@ -280,29 +280,30 @@ The application layer is out of scope of this script. As a demo purpuse only, on
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [organization_domain](variables.tf#L114) | Organization domain. | <code>string</code> | ✓ | |
|
| [organization_domain](variables.tf#L122) | Organization domain. | <code>string</code> | ✓ | |
|
||||||
| [prefix](variables.tf#L119) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
| [prefix](variables.tf#L127) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||||
| [project_config](variables.tf#L128) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ landing = string processing = string curated = string common = string }), { landing = "lnd" processing = "prc" curated = "cur" common = "cmn" } ) })">object({…})</code> | ✓ | |
|
| [project_config](variables.tf#L136) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ landing = string processing = string curated = string common = string }), { landing = "lnd" processing = "prc" curated = "cur" common = "cmn" } ) })">object({…})</code> | ✓ | |
|
||||||
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool, false) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(map(string), {}) pypi_packages = optional(map(string), {}) env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) count = optional(number, 1) } ), {}) web_server = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) }), {}) worker = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) min_count = optional(number, 1) max_count = optional(number, 3) } ), {}) }), {}) })">object({…})</code> | | <code>{}</code> |
|
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional(object({ airflow_config_overrides = optional(map(string), {}) pypi_packages = optional(map(string), {}) env_variables = optional(map(string), {}) image_version = optional(string, "composer-2-airflow-2") }), {}) workloads_config = optional(object({ scheduler = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) count = optional(number, 1) } ), {}) web_server = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) }), {}) worker = optional(object({ cpu = optional(number, 0.5) memory_gb = optional(number, 1.875) storage_gb = optional(number, 1) min_count = optional(number, 1) max_count = optional(number, 3) } ), {}) }), {}) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [data_catalog_tags](variables.tf#L55) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code title="{ "3_Confidential" = null "2_Private" = null "1_Sensitive" = null }">{…}</code> |
|
| [data_catalog_tags](variables.tf#L54) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code>map(map(list(string)))</code> | | <code title="{ "3_Confidential" = null "2_Private" = null "1_Sensitive" = null }">{…}</code> |
|
||||||
| [data_force_destroy](variables.tf#L66) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
| [data_force_destroy](variables.tf#L65) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | <code>bool</code> | | <code>false</code> |
|
||||||
| [groups](variables.tf#L72) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
| [enable_services](variables.tf#L71) | Flag to enable or disable services in the Data Platform. | <code title="object({ composer = optional(bool, true) dataproc_history_server = optional(bool, true) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [location](variables.tf#L82) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
| [groups](variables.tf#L80) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||||
| [network_config](variables.tf#L88) | Shared VPC network configurations to use. If null networks will be created in projects. | <code title="object({ host_project = optional(string) network_self_link = optional(string) subnet_self_links = optional(object({ processing_dataproc = string processing_composer = string }), null) composer_ip_ranges = optional(object({ connection_subnetwork = optional(string) cloud_sql = optional(string, "10.20.10.0/24") gke_master = optional(string, "10.20.11.0/28") pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) })">object({…})</code> | | <code>{}</code> |
|
| [location](variables.tf#L90) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
||||||
| [project_suffix](variables.tf#L152) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
| [network_config](variables.tf#L96) | Shared VPC network configurations to use. If null networks will be created in projects. | <code title="object({ host_project = optional(string) network_self_link = optional(string) subnet_self_links = optional(object({ processing_transformation = string processing_composer = string }), null) composer_ip_ranges = optional(object({ connection_subnetwork = optional(string) cloud_sql = optional(string, "10.20.10.0/24") gke_master = optional(string, "10.20.11.0/28") pods_range_name = optional(string, "pods") services_range_name = optional(string, "services") }), {}) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [region](variables.tf#L158) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [project_suffix](variables.tf#L160) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||||
| [service_encryption_keys](variables.tf#L164) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = optional(string) composer = optional(string) compute = optional(string) storage = optional(string) })">object({…})</code> | | <code>{}</code> |
|
| [region](variables.tf#L166) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
|
| [service_encryption_keys](variables.tf#L172) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = optional(string) composer = optional(string) compute = optional(string) storage = optional(string) })">object({…})</code> | | <code>{}</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
| name | description | sensitive |
|
| name | description | sensitive |
|
||||||
|---|---|:---:|
|
|---|---|:---:|
|
||||||
| [bigquery-datasets](outputs.tf#L17) | BigQuery datasets. | |
|
| [bigquery-datasets](outputs.tf#L17) | BigQuery datasets. | |
|
||||||
| [dataproc-hystory-server](outputs.tf#L24) | List of bucket names which have been assigned to the cluster. | |
|
| [dataproc-history-server](outputs.tf#L24) | List of bucket names which have been assigned to the cluster. | |
|
||||||
| [gcs-buckets](outputs.tf#L34) | GCS buckets. | ✓ |
|
| [gcs-buckets](outputs.tf#L29) | GCS buckets. | ✓ |
|
||||||
| [kms_keys](outputs.tf#L44) | Cloud MKS keys. | |
|
| [kms_keys](outputs.tf#L39) | Cloud MKS keys. | |
|
||||||
| [projects](outputs.tf#L49) | GCP Projects informations. | |
|
| [projects](outputs.tf#L44) | GCP Projects informations. | |
|
||||||
| [vpc_network](outputs.tf#L67) | VPC network. | |
|
| [vpc_network](outputs.tf#L62) | VPC network. | |
|
||||||
| [vpc_subnet](outputs.tf#L75) | VPC subnetworks. | |
|
| [vpc_subnet](outputs.tf#L70) | VPC subnetworks. | |
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -41,9 +41,9 @@ LAND_GCS = os.environ.get("LAND_GCS")
|
||||||
PHS_CLUSTER_NAME = os.environ.get("PHS_CLUSTER_NAME")
|
PHS_CLUSTER_NAME = os.environ.get("PHS_CLUSTER_NAME")
|
||||||
PROCESSING_GCS = os.environ.get("PROCESSING_GCS")
|
PROCESSING_GCS = os.environ.get("PROCESSING_GCS")
|
||||||
PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ")
|
PROCESSING_PRJ = os.environ.get("PROCESSING_PRJ")
|
||||||
PROCESSING_SA_DP = os.environ.get("PROCESSING_SA_DP")
|
PROCESSING_SA = os.environ.get("PROCESSING_SA")
|
||||||
PROCESSING_SA_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
PROCESSING_SUBNET = os.environ.get("PROCESSING_SUBNET")
|
||||||
PROCESSING_SA_VPC = os.environ.get("PROCESSING_VPC")
|
PROCESSING_VPC = os.environ.get("PROCESSING_VPC")
|
||||||
|
|
||||||
PYTHON_FILE_LOCATION = "gs://"+PROCESSING_GCS+"/pyspark_sort.py"
|
PYTHON_FILE_LOCATION = "gs://"+PROCESSING_GCS+"/pyspark_sort.py"
|
||||||
PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_CLUSTER_NAME
|
PHS_CLUSTER_PATH = "projects/"+PROCESSING_PRJ+"/regions/"+DP_REGION+"/clusters/"+PHS_CLUSTER_NAME
|
||||||
|
@ -65,8 +65,8 @@ with models.DAG(
|
||||||
batch={
|
batch={
|
||||||
"environment_config": {
|
"environment_config": {
|
||||||
"execution_config": {
|
"execution_config": {
|
||||||
"service_account": PROCESSING_SA_DP,
|
"service_account": PROCESSING_SA,
|
||||||
"subnetwork_uri": PROCESSING_SA_SUBNET
|
"subnetwork_uri": PROCESSING_SUBNET
|
||||||
},
|
},
|
||||||
"peripherals_config": {
|
"peripherals_config": {
|
||||||
"spark_history_server_config":{
|
"spark_history_server_config":{
|
||||||
|
|
|
@ -21,14 +21,9 @@ output "bigquery-datasets" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output "dataproc-hystory-server" {
|
output "dataproc-history-server" {
|
||||||
description = "List of bucket names which have been assigned to the cluster."
|
description = "List of bucket names which have been assigned to the cluster."
|
||||||
value = {
|
value = one(module.processing-dp-historyserver)
|
||||||
bucket_names = module.processing-dp-historyserver.bucket_names
|
|
||||||
http_ports = module.processing-dp-historyserver.http_ports
|
|
||||||
instance_names = module.processing-dp-historyserver.instance_names
|
|
||||||
name = module.processing-dp-historyserver.name
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
output "gcs-buckets" {
|
output "gcs-buckets" {
|
||||||
|
@ -67,7 +62,7 @@ output "projects" {
|
||||||
output "vpc_network" {
|
output "vpc_network" {
|
||||||
description = "VPC network."
|
description = "VPC network."
|
||||||
value = {
|
value = {
|
||||||
processing_dataproc = local.processing_vpc
|
processing_transformation = local.processing_vpc
|
||||||
processing_composer = local.processing_vpc
|
processing_composer = local.processing_vpc
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -75,7 +70,7 @@ output "vpc_network" {
|
||||||
output "vpc_subnet" {
|
output "vpc_subnet" {
|
||||||
description = "VPC subnetworks."
|
description = "VPC subnetworks."
|
||||||
value = {
|
value = {
|
||||||
processing_dataproc = local.processing_subnet
|
processing_transformation = local.processing_subnet
|
||||||
processing_composer = local.processing_subnet
|
processing_composer = local.processing_subnet
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
variable "composer_config" {
|
variable "composer_config" {
|
||||||
description = "Cloud Composer config."
|
description = "Cloud Composer config."
|
||||||
type = object({
|
type = object({
|
||||||
disable_deployment = optional(bool, false)
|
|
||||||
environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL")
|
environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL")
|
||||||
software_config = optional(object({
|
software_config = optional(object({
|
||||||
airflow_config_overrides = optional(map(string), {})
|
airflow_config_overrides = optional(map(string), {})
|
||||||
|
@ -69,6 +68,15 @@ variable "data_force_destroy" {
|
||||||
default = false
|
default = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "enable_services" {
|
||||||
|
description = "Flag to enable or disable services in the Data Platform."
|
||||||
|
type = object({
|
||||||
|
composer = optional(bool, true)
|
||||||
|
dataproc_history_server = optional(bool, true)
|
||||||
|
})
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
variable "groups" {
|
variable "groups" {
|
||||||
description = "User groups."
|
description = "User groups."
|
||||||
type = map(string)
|
type = map(string)
|
||||||
|
@ -91,7 +99,7 @@ variable "network_config" {
|
||||||
host_project = optional(string)
|
host_project = optional(string)
|
||||||
network_self_link = optional(string)
|
network_self_link = optional(string)
|
||||||
subnet_self_links = optional(object({
|
subnet_self_links = optional(object({
|
||||||
processing_dataproc = string
|
processing_transformation = string
|
||||||
processing_composer = string
|
processing_composer = string
|
||||||
}), null)
|
}), null)
|
||||||
composer_ip_ranges = optional(object({
|
composer_ip_ranges = optional(object({
|
||||||
|
|
Loading…
Reference in New Issue