cloud-foundation-fabric/modules/dataplex-datascan/main.tf

/**
 * Copyright 2023 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

locals {
  data_quality_spec = {
    post_scan_actions = try(
      var.data_quality_spec.post_scan_actions,
      local.factory_data.post_scan_actions,
      null
    )
    row_filter = try(
      var.data_quality_spec.row_filter,
      local.factory_data.row_filter,
      null
    )
    rules = concat(
      try(var.data_quality_spec.rules, []),
      try(local.factory_data.rules, [])
    )
    sampling_percent = try(
      var.data_quality_spec.sampling_percent,
      local.factory_data.sampling_percent,
      null
    )
  }
  prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}-"
  use_data_quality = (
    var.data_quality_spec != null ||
    var.factories_config.data_quality_spec != null
  )
}

resource "google_dataplex_datascan" "datascan" {
  project      = var.project_id
  location     = var.region
  data_scan_id = "${local.prefix}${var.name}"
  display_name = "${local.prefix}${var.name}"
  description  = var.description == null ? "Terraform Managed." : "Terraform Managed. ${var.description}"
  labels       = var.labels

  data {
    resource = var.data.resource
    entity   = var.data.entity
  }

  execution_spec {
    field = var.incremental_field
    trigger {
      dynamic "on_demand" {
        for_each = var.execution_schedule == null ? [""] : []
        content {
        }
      }
      dynamic "schedule" {
        for_each = var.execution_schedule != null ? [""] : []
        content {
          cron = var.execution_schedule
        }
      }
    }
  }

  dynamic "data_profile_spec" {
    for_each = var.data_profile_spec != null ? [""] : []
    content {
      sampling_percent = try(var.data_profile_spec.sampling_percent, null)
      row_filter       = try(var.data_profile_spec.row_filter, null)
    }
  }

  dynamic "data_quality_spec" {
    for_each = local.use_data_quality ? [""] : []
    content {
      sampling_percent = try(local.data_quality_spec.sampling_percent, null)
      row_filter       = try(local.data_quality_spec.row_filter, null)
      dynamic "post_scan_actions" {
        for_each = local.data_quality_spec.post_scan_actions != null ? [""] : []
        content {
          dynamic "bigquery_export" {
            for_each = (
              local.data_quality_spec.post_scan_actions.bigquery_export != null
              ? [""]
              : []
            )
            content {
              results_table = try(
                local.data_quality_spec.post_scan_actions.bigquery_export.results_table,
                null
              )
            }
          }
        }
      }
      dynamic "rules" {
        for_each = local.data_quality_spec.rules
        content {
          column      = try(rules.value.column, null)
          ignore_null = try(rules.value.ignore_null, null)
          dimension   = rules.value.dimension
          threshold   = try(rules.value.threshold, null)

          dynamic "non_null_expectation" {
            for_each = try(rules.value.non_null_expectation, null) != null ? [""] : []
            content {
            }
          }

          dynamic "range_expectation" {
            for_each = (
              try(rules.value.range_expectation, null) != null ? [""] : []
            )
            content {
              min_value = try(
                rules.value.range_expectation.min_value, null
              )
              max_value = try(
                rules.value.range_expectation.max_value, null
              )
              strict_min_enabled = try(
                rules.value.range_expectation.strict_min_enabled, null
              )
              strict_max_enabled = try(
                rules.value.range_expectation.strict_max_enabled, null
              )
            }
          }

          dynamic "set_expectation" {
            for_each = (
              try(rules.value.set_expectation, null) != null ? [""] : []
            )
            content {
              values = rules.value.set_expectation.values
            }
          }

          dynamic "uniqueness_expectation" {
            for_each = (
              try(rules.value.uniqueness_expectation, null) != null ? [""] : []
            )
            content {
            }
          }

          dynamic "regex_expectation" {
            for_each = (
              try(rules.value.regex_expectation, null) != null ? [""] : []
            )
            content {
              regex = rules.value.regex_expectation.regex
            }
          }

          dynamic "statistic_range_expectation" {
            for_each = (
              try(rules.value.statistic_range_expectation, null) != null ? [""] : []
            )
            content {
              min_value = try(
                rules.value.statistic_range_expectation.min_value, null
              )
              max_value = try(
                rules.value.statistic_range_expectation.max_value, null
              )
              strict_min_enabled = try(
                rules.value.statistic_range_expectation.strict_min_enabled, null
              )
              strict_max_enabled = try(
                rules.value.statistic_range_expectation.strict_max_enabled, null
              )
              statistic = rules.value.statistic_range_expectation.statistic
            }
          }

          dynamic "row_condition_expectation" {
            for_each = (
              try(rules.value.row_condition_expectation, null) != null ? [""] : []
            )
            content {
              sql_expression = rules.value.row_condition_expectation.sql_expression
            }
          }

          dynamic "table_condition_expectation" {
            for_each = (
              try(rules.value.table_condition_expectation, null) != null ? [""] : []
            )
            content {
              sql_expression = rules.value.table_condition_expectation.sql_expression
            }
          }

        }
      }
    }
  }

  lifecycle {
    precondition {
      condition = (
        length([
          for spec in [
            var.data_profile_spec,
            var.data_quality_spec,
            var.factories_config.data_quality_spec
          ] : spec if spec != null
        ]) == 1
      )
      error_message = "DataScan can only contain one of 'data_profile_spec', 'data_quality_spec', 'factories_config.data_quality_spec'."
    }
    precondition {
      condition = alltrue([
        for rule in try(local.data_quality_spec.rules, []) :
      contains(["COMPLETENESS", "ACCURACY", "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"], rule.dimension)])
      error_message = "Datascan 'dimension' field in 'data_quality_spec' must be one of ['COMPLETENESS', 'ACCURACY', 'CONSISTENCY', 'VALIDITY', 'UNIQUENESS', 'INTEGRITY']."
    }
    precondition {
      condition = alltrue([
        for rule in try(local.data_quality_spec.rules, []) :
        length([
          for k, v in rule :
          v if contains([
            "non_null_expectation",
            "range_expectation",
            "regex_expectation",
            "set_expectation",
            "uniqueness_expectation",
            "statistic_range_expectation",
            "row_condition_expectation",
            "table_condition_expectation"
          ], k) && v != null
      ]) == 1])
      error_message = "Datascan rule must contain a key that is one of ['non_null_expectation', 'range_expectation', 'regex_expectation', 'set_expectation', 'uniqueness_expectation', 'statistic_range_expectation', 'row_condition_expectation', 'table_condition_expectation]."
    }
  }
}