Fix Data platform foundation (#1992)
* FAST + Minimal DP * Fix tests --------- Co-authored-by: Julio Castillo <jccb@google.com>
This commit is contained in:
lcaggio
GitHub
parent
1ba8298b0e
commit
208902c8da
|
|
@ -45,7 +45,7 @@ module "drop-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ module "load-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ module "orch-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
@ -162,8 +162,8 @@ module "orch-cs-df-template" {
|
|||
project_id = module.orch-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "orc-cs-df-template"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
location = var.location
|
||||
storage_class = "MULTI_REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = !var.deletion_protection
|
||||
}
|
||||
|
|
@ -173,8 +173,8 @@ module "orch-cs-build-staging" {
|
|||
project_id = module.orch-project.project_id
|
||||
prefix = var.prefix
|
||||
name = "orc-cs-build-staging"
|
||||
location = var.region
|
||||
storage_class = "REGIONAL"
|
||||
location = var.location
|
||||
storage_class = "MULTI_REGIONAL"
|
||||
encryption_key = try(local.service_encryption_keys.storage, null)
|
||||
force_destroy = !var.deletion_protection
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ module "transf-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ module "dwh-lnd-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
@ -88,7 +88,7 @@ module "dwh-cur-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
@ -108,7 +108,7 @@ module "dwh-conf-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ module "common-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ module "exp-project" {
|
|||
source = "../../../modules/project"
|
||||
parent = var.project_config.parent
|
||||
billing_account = var.project_config.billing_account_id
|
||||
project_create = var.project_config.billing_account_id != null
|
||||
project_create = var.project_config.project_create
|
||||
prefix = local.use_projects ? null : var.prefix
|
||||
name = (
|
||||
local.use_projects
|
||||
|
|
|
|||
|
|
@ -281,17 +281,17 @@ terraform destroy
|
|||
|---|---|:---:|:---:|:---:|
|
||||
| [organization_domain](variables.tf#L166) | Organization domain. | <code>string</code> | ✓ | |
|
||||
| [prefix](variables.tf#L171) | Prefix used for resource names. | <code>string</code> | ✓ | |
|
||||
| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) parent = string project_ids = optional(object({ drop = string load = string orc = string trf = string dwh-lnd = string dwh-cur = string dwh-conf = string common = string exp = string }), { drop = "drp" load = "lod" orc = "orc" trf = "trf" dwh-lnd = "dwh-lnd" dwh-cur = "dwh-cur" dwh-conf = "dwh-conf" common = "cmn" exp = "exp" } ) })">object({…})</code> | ✓ | |
|
||||
| [project_config](variables.tf#L180) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | <code title="object({ billing_account_id = optional(string, null) project_create = optional(bool, true) parent = string project_ids = optional(object({ drop = string load = string orc = string trf = string dwh-lnd = string dwh-cur = string dwh-conf = string common = string exp = string }), { drop = "drp" load = "lod" orc = "orc" trf = "trf" dwh-lnd = "dwh-lnd" dwh-cur = "dwh-cur" dwh-conf = "dwh-conf" common = "cmn" exp = "exp" } ) })">object({…})</code> | ✓ | |
|
||||
| [composer_config](variables.tf#L17) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional( object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string cloud_data_lineage_integration = optional(bool, true) }), { image_version = "composer-2-airflow-2" } ) workloads_config = optional( object({ scheduler = optional( object({ cpu = number memory_gb = number storage_gb = number count = number }), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } ) web_server = optional( object({ cpu = number memory_gb = number storage_gb = number }), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } ) worker = optional( object({ cpu = number memory_gb = number storage_gb = number min_count = number max_count = number }), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } ) })) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } workloads_config = { scheduler = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } web_server = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } worker = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } } }">{…}</code> |
|
||||
| [data_catalog_tags](variables.tf#L106) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map(object({ description = optional(string) iam = optional(map(list(string)), {}) }))">map(object({…}))</code> | | <code title="{ "3_Confidential" = {} "2_Private" = {} "1_Sensitive" = {} }">{…}</code> |
|
||||
| [deletion_protection](variables.tf#L120) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | <code>bool</code> | | <code>false</code> |
|
||||
| [groups](variables.tf#L127) | User groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> |
|
||||
| [location](variables.tf#L137) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> |
|
||||
| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | <code title="object({ host_project = string network_self_link = string subnet_self_links = object({ load = string transformation = string orchestration = string }) composer_ip_ranges = object({ cloudsql = string gke_master = string }) composer_secondary_ranges = object({ pods = string services = string }) })">object({…})</code> | | <code>null</code> |
|
||||
| [project_services](variables.tf#L214) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||
| [project_suffix](variables.tf#L225) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L231) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [service_encryption_keys](variables.tf#L237) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> |
|
||||
| [project_services](variables.tf#L215) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> |
|
||||
| [project_suffix](variables.tf#L226) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> |
|
||||
| [region](variables.tf#L232) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||
| [service_encryption_keys](variables.tf#L238) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ locals {
|
|||
]
|
||||
]) : "${binding.role}-${binding.member}" => binding
|
||||
}
|
||||
use_projects = var.project_config.billing_account_id == null
|
||||
use_projects = !var.project_config.project_create
|
||||
use_shared_vpc = var.network_config != null
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
prefix = "dat-plat"
|
||||
project_config = {
|
||||
parent = "folders/1111111111"
|
||||
billing_account_id = "1111111-2222222-33333333"
|
||||
}
|
||||
organization_domain = "domain.com"
|
||||
|
|
@ -181,6 +181,7 @@ variable "project_config" {
|
|||
description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
|
||||
type = object({
|
||||
billing_account_id = optional(string, null)
|
||||
project_create = optional(bool, true)
|
||||
parent = string
|
||||
project_ids = optional(object({
|
||||
drop = string
|
||||
|
|
|
|||
|
|
@ -185,23 +185,24 @@ You can find examples in the `[demo](../../../../blueprints/data-solutions/data-
|
|||
|---|---|:---:|:---:|:---:|:---:|
|
||||
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object({ outputs_bucket = string })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | <code title="object({ id = string is_org_level = optional(bool, true) })">object({…})</code> | ✓ | | <code>0-bootstrap</code> |
|
||||
| [folder_ids](variables.tf#L108) | Folder to be used for the networking resources in folders/nnnn format. | <code title="object({ data-platform-dev = string })">object({…})</code> | ✓ | | <code>1-resman</code> |
|
||||
| [host_project_ids](variables.tf#L126) | Shared VPC project ids. | <code title="object({ dev-spoke-0 = string })">object({…})</code> | ✓ | | <code>2-networking</code> |
|
||||
| [organization](variables.tf#L156) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-globals</code> |
|
||||
| [prefix](variables.tf#L172) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | <code>string</code> | ✓ | | <code>00-globals</code> |
|
||||
| [composer_config](variables.tf#L38) | Cloud Composer configuration options. | <code title="object({ disable_deployment = optional(bool) environment_size = string software_config = object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string cloud_data_lineage_integration = optional(bool, true) }) workloads_config = object({ scheduler = object( { cpu = number memory_gb = number storage_gb = number count = number } ) web_server = object( { cpu = number memory_gb = number storage_gb = number } ) worker = object( { cpu = number memory_gb = number storage_gb = number min_count = number max_count = number } ) }) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" cloud_data_lineage_integration = true } workloads_config = null }">{…}</code> | |
|
||||
| [data_catalog_tags](variables.tf#L87) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map(object({ description = optional(string) iam = optional(map(list(string)), {}) }))">map(object({…}))</code> | | <code title="{ "3_Confidential" = {} "2_Private" = {} "1_Sensitive" = {} }">{…}</code> | |
|
||||
| [deletion_protection](variables.tf#L101) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | <code>bool</code> | | <code>true</code> | |
|
||||
| [groups-dp](variables.tf#L116) | Data Platform groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> | |
|
||||
| [location](variables.tf#L134) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> | |
|
||||
| [network_config_composer](variables.tf#L140) | Network configurations to use for Composer. | <code title="object({ cloudsql_range = string gke_master_range = string gke_pods_name = string gke_services_name = string })">object({…})</code> | | <code title="{ cloudsql_range = "192.168.254.0/24" gke_master_range = "192.168.255.0/28" gke_pods_name = "pods" gke_services_name = "services" }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L166) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [project_services](variables.tf#L182) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> | |
|
||||
| [project_suffix](variables.tf#L193) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> | |
|
||||
| [region](variables.tf#L199) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> | |
|
||||
| [service_encryption_keys](variables.tf#L205) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> | |
|
||||
| [subnet_self_links](variables.tf#L217) | Shared VPC subnet self links. | <code title="object({ dev-spoke-0 = map(string) })">object({…})</code> | | <code>null</code> | <code>2-networking</code> |
|
||||
| [vpc_self_links](variables.tf#L226) | Shared VPC self links. | <code title="object({ dev-spoke-0 = string })">object({…})</code> | | <code>null</code> | <code>2-networking</code> |
|
||||
| [folder_ids](variables.tf#L148) | Folder to be used for the networking resources in folders/nnnn format. | <code title="object({ data-platform-dev = string })">object({…})</code> | ✓ | | <code>1-resman</code> |
|
||||
| [host_project_ids](variables.tf#L166) | Shared VPC project ids. | <code title="object({ dev-spoke-0 = string })">object({…})</code> | ✓ | | <code>2-networking</code> |
|
||||
| [organization](variables.tf#L196) | Organization details. | <code title="object({ domain = string id = number customer_id = string })">object({…})</code> | ✓ | | <code>00-globals</code> |
|
||||
| [prefix](variables.tf#L212) | Unique prefix used for resource names. Not used for projects if 'project_create' is null. | <code>string</code> | ✓ | | <code>00-globals</code> |
|
||||
| [composer_config](variables.tf#L38) | Cloud Composer config. | <code title="object({ disable_deployment = optional(bool) environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL") software_config = optional( object({ airflow_config_overrides = optional(any) pypi_packages = optional(any) env_variables = optional(map(string)) image_version = string cloud_data_lineage_integration = optional(bool, true) }), { image_version = "composer-2-airflow-2" } ) workloads_config = optional( object({ scheduler = optional( object({ cpu = number memory_gb = number storage_gb = number count = number }), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } ) web_server = optional( object({ cpu = number memory_gb = number storage_gb = number }), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } ) worker = optional( object({ cpu = number memory_gb = number storage_gb = number min_count = number max_count = number }), { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } ) })) })">object({…})</code> | | <code title="{ environment_size = "ENVIRONMENT_SIZE_SMALL" software_config = { image_version = "composer-2-airflow-2" } workloads_config = { scheduler = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 count = 1 } web_server = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 } worker = { cpu = 0.5 memory_gb = 1.875 storage_gb = 1 min_count = 1 max_count = 3 } } }">{…}</code> | |
|
||||
| [data_catalog_tags](variables.tf#L127) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | <code title="map(object({ description = optional(string) iam = optional(map(list(string)), {}) }))">map(object({…}))</code> | | <code title="{ "3_Confidential" = {} "2_Private" = {} "1_Sensitive" = {} }">{…}</code> | |
|
||||
| [deletion_protection](variables.tf#L141) | Prevent Terraform from destroying data storage resources (storage buckets, GKE clusters, CloudSQL instances) in this blueprint. When this field is set in Terraform state, a terraform destroy or terraform apply that would delete data storage resources will fail. | <code>bool</code> | | <code>true</code> | |
|
||||
| [groups_dp](variables.tf#L156) | Data Platform groups. | <code>map(string)</code> | | <code title="{ data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" data-security = "gcp-data-security" }">{…}</code> | |
|
||||
| [location](variables.tf#L174) | Location used for multi-regional resources. | <code>string</code> | | <code>"eu"</code> | |
|
||||
| [network_config_composer](variables.tf#L180) | Network configurations to use for Composer. | <code title="object({ cloudsql_range = string gke_master_range = string gke_pods_name = string gke_services_name = string })">object({…})</code> | | <code title="{ cloudsql_range = "192.168.254.0/24" gke_master_range = "192.168.255.0/28" gke_pods_name = "pods" gke_services_name = "services" }">{…}</code> | |
|
||||
| [outputs_location](variables.tf#L206) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | <code>string</code> | | <code>null</code> | |
|
||||
| [project_config](variables.tf#L222) | Provide projects configuration. | <code title="object({ project_create = optional(bool, true) project_ids = optional(object({ drop = string load = string orc = string trf = string dwh-lnd = string dwh-cur = string dwh-conf = string common = string exp = string }) ) })">object({…})</code> | | <code>{}</code> | |
|
||||
| [project_services](variables.tf#L242) | List of core services enabled on all projects. | <code>list(string)</code> | | <code title="[ "cloudresourcemanager.googleapis.com", "iam.googleapis.com", "serviceusage.googleapis.com", "stackdriver.googleapis.com" ]">[…]</code> | |
|
||||
| [project_suffix](variables.tf#L253) | Suffix used only for project ids. | <code>string</code> | | <code>null</code> | |
|
||||
| [region](variables.tf#L259) | Region used for regional resources. | <code>string</code> | | <code>"europe-west1"</code> | |
|
||||
| [service_encryption_keys](variables.tf#L265) | Cloud KMS to use to encrypt different services. Key location should match service region. | <code title="object({ bq = string composer = string dataflow = string storage = string pubsub = string })">object({…})</code> | | <code>null</code> | |
|
||||
| [subnet_self_links](variables.tf#L277) | Shared VPC subnet self links. | <code title="object({ dev-spoke-0 = map(string) })">object({…})</code> | | <code>null</code> | <code>2-networking</code> |
|
||||
| [vpc_self_links](variables.tf#L286) | Shared VPC self links. | <code title="object({ dev-spoke-0 = string })">object({…})</code> | | <code>null</code> | <code>2-networking</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -23,9 +23,11 @@ module "data-platform" {
|
|||
data_catalog_tags = var.data_catalog_tags
|
||||
project_config = {
|
||||
billing_account_id = var.billing_account.id
|
||||
project_create = var.project_config.project_create
|
||||
parent = var.folder_ids.data-platform-dev
|
||||
project_ids = var.project_config.project_ids
|
||||
}
|
||||
groups = var.groups-dp
|
||||
groups = var.groups_dp
|
||||
location = var.location
|
||||
network_config = {
|
||||
host_project = var.host_project_ids.dev-spoke-0
|
||||
|
|
|
|||
|
|
@ -36,51 +36,91 @@ variable "billing_account" {
|
|||
}
|
||||
|
||||
variable "composer_config" {
|
||||
description = "Cloud Composer configuration options."
|
||||
description = "Cloud Composer config."
|
||||
type = object({
|
||||
disable_deployment = optional(bool)
|
||||
environment_size = string
|
||||
software_config = object({
|
||||
airflow_config_overrides = optional(any)
|
||||
pypi_packages = optional(any)
|
||||
env_variables = optional(map(string))
|
||||
image_version = string
|
||||
cloud_data_lineage_integration = optional(bool, true)
|
||||
})
|
||||
workloads_config = object({
|
||||
scheduler = object(
|
||||
{
|
||||
cpu = number
|
||||
memory_gb = number
|
||||
storage_gb = number
|
||||
count = number
|
||||
}
|
||||
)
|
||||
web_server = object(
|
||||
{
|
||||
cpu = number
|
||||
memory_gb = number
|
||||
storage_gb = number
|
||||
}
|
||||
)
|
||||
worker = object(
|
||||
{
|
||||
cpu = number
|
||||
memory_gb = number
|
||||
storage_gb = number
|
||||
min_count = number
|
||||
max_count = number
|
||||
}
|
||||
)
|
||||
})
|
||||
environment_size = optional(string, "ENVIRONMENT_SIZE_SMALL")
|
||||
software_config = optional(
|
||||
object({
|
||||
airflow_config_overrides = optional(any)
|
||||
pypi_packages = optional(any)
|
||||
env_variables = optional(map(string))
|
||||
image_version = string
|
||||
cloud_data_lineage_integration = optional(bool, true)
|
||||
}),
|
||||
{ image_version = "composer-2-airflow-2" }
|
||||
)
|
||||
workloads_config = optional(
|
||||
object({
|
||||
scheduler = optional(
|
||||
object({
|
||||
cpu = number
|
||||
memory_gb = number
|
||||
storage_gb = number
|
||||
count = number
|
||||
}),
|
||||
{
|
||||
cpu = 0.5
|
||||
memory_gb = 1.875
|
||||
storage_gb = 1
|
||||
count = 1
|
||||
}
|
||||
)
|
||||
web_server = optional(
|
||||
object({
|
||||
cpu = number
|
||||
memory_gb = number
|
||||
storage_gb = number
|
||||
}),
|
||||
{
|
||||
cpu = 0.5
|
||||
memory_gb = 1.875
|
||||
storage_gb = 1
|
||||
}
|
||||
)
|
||||
worker = optional(
|
||||
object({
|
||||
cpu = number
|
||||
memory_gb = number
|
||||
storage_gb = number
|
||||
min_count = number
|
||||
max_count = number
|
||||
}),
|
||||
{
|
||||
cpu = 0.5
|
||||
memory_gb = 1.875
|
||||
storage_gb = 1
|
||||
min_count = 1
|
||||
max_count = 3
|
||||
}
|
||||
)
|
||||
}))
|
||||
})
|
||||
default = {
|
||||
environment_size = "ENVIRONMENT_SIZE_SMALL"
|
||||
software_config = {
|
||||
image_version = "composer-2-airflow-2"
|
||||
cloud_data_lineage_integration = true
|
||||
image_version = "composer-2-airflow-2"
|
||||
}
|
||||
workloads_config = {
|
||||
scheduler = {
|
||||
cpu = 0.5
|
||||
memory_gb = 1.875
|
||||
storage_gb = 1
|
||||
count = 1
|
||||
}
|
||||
web_server = {
|
||||
cpu = 0.5
|
||||
memory_gb = 1.875
|
||||
storage_gb = 1
|
||||
}
|
||||
worker = {
|
||||
cpu = 0.5
|
||||
memory_gb = 1.875
|
||||
storage_gb = 1
|
||||
min_count = 1
|
||||
max_count = 3
|
||||
}
|
||||
}
|
||||
workloads_config = null
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -113,7 +153,7 @@ variable "folder_ids" {
|
|||
})
|
||||
}
|
||||
|
||||
variable "groups-dp" {
|
||||
variable "groups_dp" {
|
||||
description = "Data Platform groups."
|
||||
type = map(string)
|
||||
default = {
|
||||
|
|
@ -179,6 +219,26 @@ variable "prefix" {
|
|||
}
|
||||
}
|
||||
|
||||
variable "project_config" {
|
||||
description = "Provide projects configuration."
|
||||
type = object({
|
||||
project_create = optional(bool, true)
|
||||
project_ids = optional(object({
|
||||
drop = string
|
||||
load = string
|
||||
orc = string
|
||||
trf = string
|
||||
dwh-lnd = string
|
||||
dwh-cur = string
|
||||
dwh-conf = string
|
||||
common = string
|
||||
exp = string
|
||||
})
|
||||
)
|
||||
})
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "project_services" {
|
||||
description = "List of core services enabled on all projects."
|
||||
type = list(string)
|
||||
|
|
|
|||
Loading…
Reference in New Issue