Updated the DataQualitySpec for Dataplex Datascan (#2008)
* Updated the DataQualitySpec for Dataplex Datascan * Fix linting --------- Co-authored-by: Ludovico Magnocavallo <ludomagno@google.com> Co-authored-by: Julio Castillo <jccb@google.com>
This commit is contained in:
shourya116
GitHub
parent
37fc16ab42
commit
7b58114d65
|
|
@ -431,21 +431,21 @@ module "dataplex-datascan" {
|
|||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | <code title="object({ entity = optional(string) resource = optional(string) })">object({…})</code> | ✓ | |
|
||||
| [name](variables.tf#L157) | Name of Dataplex Scan. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L168) | The ID of the project where the Dataplex DataScan will be created. | <code>string</code> | ✓ | |
|
||||
| [region](variables.tf#L173) | Region for the Dataplex DataScan. | <code>string</code> | ✓ | |
|
||||
| [name](variables.tf#L162) | Name of Dataplex Scan. | <code>string</code> | ✓ | |
|
||||
| [project_id](variables.tf#L173) | The ID of the project where the Dataplex DataScan will be created. | <code>string</code> | ✓ | |
|
||||
| [region](variables.tf#L178) | Region for the Dataplex DataScan. | <code>string</code> | ✓ | |
|
||||
| [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | <code title="object({ sampling_percent = optional(number) row_filter = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||
| [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | <code title="object({ sampling_percent = optional(number) row_filter = optional(string) rules = list(object({ column = optional(string) ignore_null = optional(bool, null) dimension = string threshold = optional(number) non_null_expectation = optional(object({})) range_expectation = optional(object({ min_value = optional(number) max_value = optional(number) strict_min_enabled = optional(bool) strict_max_enabled = optional(bool) })) regex_expectation = optional(object({ regex = string })) set_expectation = optional(object({ values = list(string) })) uniqueness_expectation = optional(object({})) statistic_range_expectation = optional(object({ statistic = string min_value = optional(number) max_value = optional(number) strict_min_enabled = optional(bool) strict_max_enabled = optional(bool) })) row_condition_expectation = optional(object({ sql_expression = string })) table_condition_expectation = optional(object({ sql_expression = string })) })) })">object({…})</code> | | <code>null</code> |
|
||||
| [data_quality_spec_file](variables.tf#L80) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | <code title="object({ path = string })">object({…})</code> | | <code>null</code> |
|
||||
| [description](variables.tf#L88) | Custom description for DataScan. | <code>string</code> | | <code>null</code> |
|
||||
| [execution_schedule](variables.tf#L94) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | <code>string</code> | | <code>null</code> |
|
||||
| [group_iam](variables.tf#L100) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [iam](variables.tf#L107) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [iam_bindings](variables.tf#L114) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map(object({ members = list(string) role = string condition = optional(object({ expression = string title = string description = optional(string) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [iam_bindings_additive](variables.tf#L129) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map(object({ member = string role = string condition = optional(object({ expression = string title = string description = optional(string) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [incremental_field](variables.tf#L144) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | <code>string</code> | | <code>null</code> |
|
||||
| [labels](variables.tf#L150) | Resource labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||
| [prefix](variables.tf#L162) | Optional prefix used to generate Dataplex DataScan ID. | <code>string</code> | | <code>null</code> |
|
||||
| [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | <code title="object({ sampling_percent = optional(number) row_filter = optional(string) post_scan_actions = optional(object({ bigquery_export = optional(object({ results_table = optional(string) })) })) rules = list(object({ column = optional(string) ignore_null = optional(bool, null) dimension = string threshold = optional(number) non_null_expectation = optional(object({})) range_expectation = optional(object({ min_value = optional(number) max_value = optional(number) strict_min_enabled = optional(bool) strict_max_enabled = optional(bool) })) regex_expectation = optional(object({ regex = string })) set_expectation = optional(object({ values = list(string) })) uniqueness_expectation = optional(object({})) statistic_range_expectation = optional(object({ statistic = string min_value = optional(number) max_value = optional(number) strict_min_enabled = optional(bool) strict_max_enabled = optional(bool) })) row_condition_expectation = optional(object({ sql_expression = string })) table_condition_expectation = optional(object({ sql_expression = string })) })) })">object({…})</code> | | <code>null</code> |
|
||||
| [data_quality_spec_file](variables.tf#L85) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | <code title="object({ path = string })">object({…})</code> | | <code>null</code> |
|
||||
| [description](variables.tf#L93) | Custom description for DataScan. | <code>string</code> | | <code>null</code> |
|
||||
| [execution_schedule](variables.tf#L99) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | <code>string</code> | | <code>null</code> |
|
||||
| [group_iam](variables.tf#L105) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [iam](variables.tf#L112) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> |
|
||||
| [iam_bindings](variables.tf#L119) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map(object({ members = list(string) role = string condition = optional(object({ expression = string title = string description = optional(string) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [iam_bindings_additive](variables.tf#L134) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map(object({ member = string role = string condition = optional(object({ expression = string title = string description = optional(string) })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||
| [incremental_field](variables.tf#L149) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | <code>string</code> | | <code>null</code> |
|
||||
| [labels](variables.tf#L155) | Resource labels. | <code>map(string)</code> | | <code>{}</code> |
|
||||
| [prefix](variables.tf#L167) | Optional prefix used to generate Dataplex DataScan ID. | <code>string</code> | | <code>null</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
|
|
|
|||
|
|
@ -17,9 +17,10 @@
|
|||
locals {
|
||||
prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}-"
|
||||
_file_data_quality_spec = var.data_quality_spec_file == null ? null : {
|
||||
sampling_percent = try(local._file_data_quality_spec_raw.samplingPercent, local._file_data_quality_spec_raw.sampling_percent, null)
|
||||
row_filter = try(local._file_data_quality_spec_raw.rowFilter, local._file_data_quality_spec_raw.row_filter, null)
|
||||
rules = local._parsed_rules
|
||||
sampling_percent = try(local._file_data_quality_spec_raw.samplingPercent, local._file_data_quality_spec_raw.sampling_percent, null)
|
||||
row_filter = try(local._file_data_quality_spec_raw.rowFilter, local._file_data_quality_spec_raw.row_filter, null)
|
||||
rules = local._parsed_rules
|
||||
post_scan_actions = try(local._file_data_quality_spec_raw.postScanActions, local._file_data_quality_spec_raw.post_scan_actions, null)
|
||||
}
|
||||
data_quality_spec = (
|
||||
var.data_quality_spec != null || var.data_quality_spec_file != null ?
|
||||
|
|
@ -71,6 +72,17 @@ resource "google_dataplex_datascan" "datascan" {
|
|||
content {
|
||||
sampling_percent = try(local.data_quality_spec.sampling_percent, null)
|
||||
row_filter = try(local.data_quality_spec.row_filter, null)
|
||||
dynamic "post_scan_actions" {
|
||||
for_each = local.data_quality_spec.post_scan_actions != null ? [""] : []
|
||||
content {
|
||||
dynamic "bigquery_export" {
|
||||
for_each = local.data_quality_spec.post_scan_actions.bigquery_export != null ? [""] : []
|
||||
content {
|
||||
results_table = try(local.data_quality_spec.post_scan_actions.bigquery_export.results_table, null)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dynamic "rules" {
|
||||
for_each = local.data_quality_spec.rules
|
||||
content {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ variable "data_quality_spec" {
|
|||
type = object({
|
||||
sampling_percent = optional(number)
|
||||
row_filter = optional(string)
|
||||
post_scan_actions = optional(object({
|
||||
bigquery_export = optional(object({
|
||||
results_table = optional(string)
|
||||
}))
|
||||
}))
|
||||
rules = list(object({
|
||||
column = optional(string)
|
||||
ignore_null = optional(bool, null)
|
||||
|
|
|
|||
Loading…
Reference in New Issue