feat: new version of projects-data-source based on AssetInventory ds
This commit is contained in:
Aleksandr Averbukh
parent
a119ce2d88
commit
a497aef707
|
|
@ -1,9 +1,14 @@
|
|||
# Projects Data Source Module
|
||||
|
||||
This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects and folders under a specific `parent` recursively.
|
||||
This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service.
|
||||
|
||||
A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically.
|
||||
|
||||
### IAM Permissions required
|
||||
|
||||
- `roles/cloudasset.viewer` on the `parent` level or above
|
||||
|
||||
|
||||
## Examples
|
||||
|
||||
### All projects in my org
|
||||
|
|
@ -15,11 +20,7 @@ module "my-org" {
|
|||
}
|
||||
|
||||
output "projects" {
|
||||
value = module.my-org.projects
|
||||
}
|
||||
|
||||
output "folders" {
|
||||
value = module.my-org.folders
|
||||
value = module.my-org.projects_numbers
|
||||
}
|
||||
|
||||
# tftest skip (uses data sources)
|
||||
|
|
@ -31,34 +32,65 @@ output "folders" {
|
|||
module "my-dev" {
|
||||
source = "./fabric/modules/projects-data-source"
|
||||
parent = "folders/123456789"
|
||||
filter = "labels.env:DEV lifecycleState:ACTIVE"
|
||||
query = "labels.env:DEV state:ACTIVE"
|
||||
}
|
||||
|
||||
output "dev-projects" {
|
||||
value = module.my-dev.projects
|
||||
}
|
||||
|
||||
output "dev-folders" {
|
||||
value = module.my-dev.folders
|
||||
# tftest skip (uses data sources)
|
||||
```
|
||||
|
||||
### Projects under org with folder/project exclusions
|
||||
```hcl
|
||||
module "my-filtered" {
|
||||
source = "./fabric/modules/projects-data-source"
|
||||
parent = "organizations/123456789"
|
||||
ignore_projects = [
|
||||
"sandbox-*", # wildcard ignore
|
||||
"project-full-id", # specific project id
|
||||
"0123456789" # specific project number
|
||||
]
|
||||
|
||||
include_projects = [
|
||||
"sandbox-114", # include specific project which was excluded by wildcard
|
||||
"415216609246" # include specific project which was excluded by wildcard (by project number)
|
||||
]
|
||||
|
||||
ignore_folders = [ # subfolders are ingoner as well
|
||||
"343991594985",
|
||||
"437102807785",
|
||||
"345245235245"
|
||||
]
|
||||
query = "state:ACTIVE"
|
||||
}
|
||||
|
||||
output "filtered-projects" {
|
||||
value = module.my-filtered.projects
|
||||
}
|
||||
|
||||
# tftest skip (uses data sources)
|
||||
|
||||
```
|
||||
|
||||
<!-- BEGIN TFDOC -->
|
||||
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [parent](variables.tf#L23) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | <code>string</code> | ✓ | |
|
||||
| [filter](variables.tf#L17) | A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters). | <code>string</code> | | <code>"lifecycleState:ACTIVE"</code> |
|
||||
| [parent](variables.tf#L17) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | <code>string</code> | ✓ | |
|
||||
| [ignore_folders](variables.tf#L58) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | <code>list(string)</code> | | <code>[]</code> |
|
||||
| [ignore_projects](variables.tf#L32) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | <code>list(string)</code> | | <code>[]</code> |
|
||||
| [include_projects](variables.tf#L44) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | <code>list(string)</code> | | <code>[]</code> |
|
||||
| [query](variables.tf#L26) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | <code>string</code> | | <code>"state:ACTIVE"</code> |
|
||||
|
||||
## Outputs
|
||||
|
||||
| name | description | sensitive |
|
||||
|---|---|:---:|
|
||||
| [folders](outputs.tf#L17) | Map of folders attributes keyed by folder id. | |
|
||||
| [project_numbers](outputs.tf#L22) | List of project numbers. | |
|
||||
| [projects](outputs.tf#L27) | Map of projects attributes keyed by projects id. | |
|
||||
| [project_numbers](outputs.tf#L17) | List of project numbers. | |
|
||||
| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. | |
|
||||
|
||||
<!-- END TFDOC -->
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
* Copyright 2023 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -15,129 +15,27 @@
|
|||
*/
|
||||
|
||||
locals {
|
||||
folders_l1_map = { for item in data.google_folders.folders_l1.folders : item.name => item }
|
||||
|
||||
folders_l2_map = merge([
|
||||
for _, v in data.google_folders.folders_l2 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l3_map = merge([
|
||||
for _, v in data.google_folders.folders_l3 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l4_map = merge([
|
||||
for _, v in data.google_folders.folders_l4 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l5_map = merge([
|
||||
for _, v in data.google_folders.folders_l5 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l6_map = merge([
|
||||
for _, v in data.google_folders.folders_l6 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l7_map = merge([
|
||||
for _, v in data.google_folders.folders_l7 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l8_map = merge([
|
||||
for _, v in data.google_folders.folders_l8 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l9_map = merge([
|
||||
for _, v in data.google_folders.folders_l9 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
folders_l10_map = merge([
|
||||
for _, v in data.google_folders.folders_l10 :
|
||||
{ for item in v.folders : item.name => item }
|
||||
]...)
|
||||
|
||||
all_folders = merge(
|
||||
local.folders_l1_map,
|
||||
local.folders_l2_map,
|
||||
local.folders_l3_map,
|
||||
local.folders_l4_map,
|
||||
local.folders_l5_map,
|
||||
local.folders_l6_map,
|
||||
local.folders_l7_map,
|
||||
local.folders_l8_map,
|
||||
local.folders_l9_map,
|
||||
local.folders_l10_map
|
||||
_ignore_folder_numbers = [for folder_id in var.ignore_folders: trimprefix(folder_id, "folders/")]
|
||||
_ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers))
|
||||
query = var.query != "" ? (
|
||||
format("%s%s", var.query, local._ignore_folders_query)
|
||||
) : (
|
||||
format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND "))
|
||||
)
|
||||
|
||||
parent_ids = toset(concat(
|
||||
[split("/", var.parent)[1]],
|
||||
[for k, _ in local.all_folders : split("/", k)[1]]
|
||||
))
|
||||
|
||||
projects = merge([
|
||||
for _, v in data.google_projects.projects :
|
||||
{ for item in v.projects : item.project_id => item }
|
||||
]...)
|
||||
ignore_patterns = [for item in var.ignore_projects: "^${replace(item, "*", ".*")}$"]
|
||||
ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$"
|
||||
projects_after_ignore = [ for item in data.google_cloud_asset_resources_search_all.projects.results : item if (
|
||||
length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0
|
||||
) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/"))
|
||||
]
|
||||
}
|
||||
|
||||
# 10 datasources are used to cover 10 possible nested layers in GCP organization hirerarcy.
|
||||
data "google_folders" "folders_l1" {
|
||||
parent_id = var.parent
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l2" {
|
||||
for_each = local.folders_l1_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l3" {
|
||||
for_each = local.folders_l2_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l4" {
|
||||
for_each = local.folders_l3_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l5" {
|
||||
for_each = local.folders_l4_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l6" {
|
||||
for_each = local.folders_l5_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l7" {
|
||||
for_each = local.folders_l6_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l8" {
|
||||
for_each = local.folders_l7_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l9" {
|
||||
for_each = local.folders_l8_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
data "google_folders" "folders_l10" {
|
||||
for_each = local.folders_l9_map
|
||||
parent_id = each.value.name
|
||||
}
|
||||
|
||||
# Getting all projects parented by any of the folders in the tree including root prg/folder provided by `parent` variable.
|
||||
data "google_projects" "projects" {
|
||||
for_each = local.parent_ids
|
||||
filter = "parent.id:${each.value} ${var.filter}"
|
||||
data google_cloud_asset_resources_search_all projects {
|
||||
provider = google-beta
|
||||
scope = var.parent
|
||||
asset_types = [
|
||||
"cloudresourcemanager.googleapis.com/Project"
|
||||
]
|
||||
query = local.query
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
* Copyright 2023 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -14,17 +14,12 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
output "folders" {
|
||||
description = "Map of folders attributes keyed by folder id."
|
||||
value = local.all_folders
|
||||
}
|
||||
|
||||
output "project_numbers" {
|
||||
description = "List of project numbers."
|
||||
value = [for _, v in local.projects : v.number]
|
||||
value = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")]
|
||||
}
|
||||
|
||||
output "projects" {
|
||||
description = "Map of projects attributes keyed by projects id."
|
||||
value = local.projects
|
||||
description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format."
|
||||
value = local.projects_after_ignore
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
* Copyright 2023 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -14,12 +14,6 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "filter" {
|
||||
description = "A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters)."
|
||||
type = string
|
||||
default = "lifecycleState:ACTIVE"
|
||||
}
|
||||
|
||||
variable "parent" {
|
||||
description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format."
|
||||
type = string
|
||||
|
|
@ -28,3 +22,47 @@ variable "parent" {
|
|||
error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
|
||||
}
|
||||
}
|
||||
|
||||
variable "query" {
|
||||
description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)."
|
||||
type = string
|
||||
default = "state:ACTIVE"
|
||||
}
|
||||
|
||||
variable "ignore_projects" {
|
||||
description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output."
|
||||
type = list(string)
|
||||
default = []
|
||||
# example
|
||||
#ignore_projects = [
|
||||
# "dev-proj-1",
|
||||
# "uat-proj-2",
|
||||
# "0123456789",
|
||||
# "prd-proj-*"
|
||||
#]
|
||||
}
|
||||
variable "include_projects" {
|
||||
description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries."
|
||||
type = list(string)
|
||||
default = []
|
||||
# example excluding all the projects starting with "prf-" except "prd-123457"
|
||||
#ignore_projects = [
|
||||
# "prd-*"
|
||||
#]
|
||||
#include_projects = [
|
||||
# "prd-123457",
|
||||
# "0123456789"
|
||||
#]
|
||||
}
|
||||
|
||||
variable "ignore_folders" {
|
||||
description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable."
|
||||
type = list(string)
|
||||
default = []
|
||||
# example exlusing a folder
|
||||
# ignore_folders = [
|
||||
# "folders/0123456789",
|
||||
# "2345678901"
|
||||
# ]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2022 Google LLC
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
|
|||
Loading…
Reference in New Issue