2023-07-29 05:31:18 -07:00
|
|
|
# Dataplex instance with lake, zone & assests
|
2023-05-05 00:26:46 -07:00
|
|
|
|
2023-07-29 05:31:18 -07:00
|
|
|
This module manages the creation of Dataplex instance along with lake, zone & assets in single regions.
|
2023-05-05 00:26:46 -07:00
|
|
|
|
|
|
|
## Simple example
|
|
|
|
|
2023-07-29 05:31:18 -07:00
|
|
|
This example shows how to setup a Dataplex instance, lake, zone & asset creation in GCP project.
|
2023-05-05 00:26:46 -07:00
|
|
|
|
|
|
|
```hcl
|
|
|
|
|
|
|
|
module "dataplex" {
|
2023-07-29 05:31:18 -07:00
|
|
|
source = "./fabric/modules/dataplex"
|
2023-05-05 00:26:46 -07:00
|
|
|
name = "terraform-lake"
|
|
|
|
prefix = "test"
|
|
|
|
project_id = "myproject"
|
|
|
|
region = "europe-west2"
|
|
|
|
zones = {
|
2023-07-24 01:52:07 -07:00
|
|
|
landing = {
|
2023-05-05 00:26:46 -07:00
|
|
|
type = "RAW"
|
|
|
|
discovery = true
|
|
|
|
assets = {
|
2023-07-24 01:52:07 -07:00
|
|
|
gcs_1 = {
|
|
|
|
resource_name = "gcs_bucket"
|
2023-05-05 00:26:46 -07:00
|
|
|
cron_schedule = "15 15 * * *"
|
|
|
|
discovery_spec_enabled = true
|
|
|
|
resource_spec_type = "STORAGE_BUCKET"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
},
|
2023-07-24 01:52:07 -07:00
|
|
|
curated = {
|
2023-05-05 00:26:46 -07:00
|
|
|
type = "CURATED"
|
2023-07-24 01:52:07 -07:00
|
|
|
discovery = false
|
|
|
|
assets = {
|
|
|
|
bq_1 = {
|
|
|
|
resource_name = "bq_dataset"
|
|
|
|
cron_schedule = null
|
|
|
|
discovery_spec_enabled = false
|
|
|
|
resource_spec_type = "BIGQUERY_DATASET"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# tftest modules=1 resources=5
|
|
|
|
```
|
|
|
|
|
|
|
|
## IAM
|
|
|
|
|
2023-07-29 05:31:18 -07:00
|
|
|
This example shows how to setup a Dataplex instance, lake, zone & asset creation in GCP project assigning IAM roles at lake and zone level.
|
2023-07-24 01:52:07 -07:00
|
|
|
|
|
|
|
```hcl
|
|
|
|
|
|
|
|
module "dataplex" {
|
2023-07-29 05:31:18 -07:00
|
|
|
source = "./fabric/modules/dataplex"
|
2023-07-24 01:52:07 -07:00
|
|
|
name = "lake"
|
|
|
|
prefix = "test"
|
|
|
|
project_id = "myproject"
|
|
|
|
region = "europe-west2"
|
|
|
|
iam = {
|
|
|
|
"roles/dataplex.viewer" = [
|
|
|
|
"group:analysts@example.com",
|
|
|
|
"group:analysts_sensitive@example.com"
|
|
|
|
]
|
|
|
|
}
|
|
|
|
zones = {
|
|
|
|
landing = {
|
|
|
|
type = "RAW"
|
2023-05-05 00:26:46 -07:00
|
|
|
discovery = true
|
|
|
|
assets = {
|
2023-07-24 01:52:07 -07:00
|
|
|
gcs_1 = {
|
|
|
|
resource_name = "gcs_bucket"
|
2023-05-05 00:26:46 -07:00
|
|
|
cron_schedule = "15 15 * * *"
|
|
|
|
discovery_spec_enabled = true
|
|
|
|
resource_spec_type = "STORAGE_BUCKET"
|
|
|
|
}
|
|
|
|
}
|
2023-07-24 01:52:07 -07:00
|
|
|
},
|
|
|
|
curated = {
|
|
|
|
type = "CURATED"
|
|
|
|
discovery = false
|
|
|
|
iam = {
|
|
|
|
"roles/viewer" = [
|
|
|
|
"group:analysts@example.com",
|
|
|
|
"group:analysts_sensitive@example.com"
|
|
|
|
]
|
|
|
|
"roles/dataplex.dataReader" = [
|
|
|
|
"group:analysts@example.com",
|
|
|
|
"group:analysts_sensitive@example.com"
|
|
|
|
]
|
|
|
|
}
|
|
|
|
assets = {
|
|
|
|
bq_1 = {
|
|
|
|
resource_name = "bq_dataset"
|
|
|
|
cron_schedule = null
|
|
|
|
discovery_spec_enabled = false
|
|
|
|
resource_spec_type = "BIGQUERY_DATASET"
|
|
|
|
}
|
|
|
|
}
|
2023-05-05 00:26:46 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-24 01:52:07 -07:00
|
|
|
# tftest modules=1 resources=8
|
2023-05-05 00:26:46 -07:00
|
|
|
```
|
2023-07-24 01:52:07 -07:00
|
|
|
|
2023-05-05 00:26:46 -07:00
|
|
|
## TODO
|
|
|
|
|
|
|
|
- [ ] support multi-regions
|
|
|
|
<!-- BEGIN TFDOC -->
|
|
|
|
|
|
|
|
## Variables
|
|
|
|
|
|
|
|
| name | description | type | required | default |
|
|
|
|
|---|---|:---:|:---:|:---:|
|
2023-07-24 01:52:07 -07:00
|
|
|
| [name](variables.tf#L30) | Name of Dataplex Lake. | <code>string</code> | ✓ | |
|
|
|
|
| [project_id](variables.tf#L41) | The ID of the project where this Dataplex Lake will be created. | <code>string</code> | ✓ | |
|
|
|
|
| [region](variables.tf#L46) | Region of the Dataplax Lake. | <code>string</code> | ✓ | |
|
|
|
|
| [zones](variables.tf#L51) | Dataplex lake zones, such as `RAW` and `CURATED`. | <code title="map(object({ type = string discovery = optional(bool, true) iam = optional(map(list(string)), null) assets = map(object({ resource_name = string resource_project = optional(string) cron_schedule = optional(string, "15 15 * * *") discovery_spec_enabled = optional(bool, true) resource_spec_type = optional(string, "STORAGE_BUCKET") })) }))">map(object({…}))</code> | ✓ | |
|
|
|
|
| [iam](variables.tf#L17) | Dataplex lake IAM bindings in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
|
|
|
| [location_type](variables.tf#L24) | The location type of the Dataplax Lake. | <code>string</code> | | <code>"SINGLE_REGION"</code> |
|
|
|
|
| [prefix](variables.tf#L35) | Optional prefix used to generate Dataplex Lake. | <code>string</code> | | <code>null</code> |
|
2023-05-05 00:26:46 -07:00
|
|
|
|
|
|
|
## Outputs
|
|
|
|
|
|
|
|
| name | description | sensitive |
|
|
|
|
|---|---|:---:|
|
|
|
|
| [assets](outputs.tf#L17) | Assets attached to the lake of Dataplex Lake. | |
|
2023-06-02 07:07:22 -07:00
|
|
|
| [id](outputs.tf#L22) | Fully qualified Dataplex Lake id. | |
|
|
|
|
| [lake](outputs.tf#L27) | The lake name of Dataplex Lake. | |
|
|
|
|
| [zones](outputs.tf#L32) | The zone name of Dataplex Lake. | |
|
2023-05-05 00:26:46 -07:00
|
|
|
|
|
|
|
<!-- END TFDOC -->
|