Merge pull request #1163 from GoogleCloudPlatform/projects-ds-new-version
Projects-data-source module new version
This commit is contained in:
commit
0d0a2b4f81
|
@ -1,9 +1,14 @@
|
||||||
# Projects Data Source Module
|
# Projects Data Source Module
|
||||||
|
|
||||||
This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects and folders under a specific `parent` recursively.
|
This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service.
|
||||||
|
|
||||||
A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically.
|
A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically.
|
||||||
|
|
||||||
|
### IAM Permissions required
|
||||||
|
|
||||||
|
- `roles/cloudasset.viewer` on the `parent` level or above
|
||||||
|
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
### All projects in my org
|
### All projects in my org
|
||||||
|
@ -14,12 +19,8 @@ module "my-org" {
|
||||||
parent = "organizations/123456789"
|
parent = "organizations/123456789"
|
||||||
}
|
}
|
||||||
|
|
||||||
output "projects" {
|
output "project_numbers" {
|
||||||
value = module.my-org.projects
|
value = module.my-org.project_numbers
|
||||||
}
|
|
||||||
|
|
||||||
output "folders" {
|
|
||||||
value = module.my-org.folders
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# tftest skip (uses data sources)
|
# tftest skip (uses data sources)
|
||||||
|
@ -31,18 +32,46 @@ output "folders" {
|
||||||
module "my-dev" {
|
module "my-dev" {
|
||||||
source = "./fabric/modules/projects-data-source"
|
source = "./fabric/modules/projects-data-source"
|
||||||
parent = "folders/123456789"
|
parent = "folders/123456789"
|
||||||
filter = "labels.env:DEV lifecycleState:ACTIVE"
|
query = "labels.env:DEV state:ACTIVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
output "dev-projects" {
|
output "dev-projects" {
|
||||||
value = module.my-dev.projects
|
value = module.my-dev.projects
|
||||||
}
|
}
|
||||||
|
|
||||||
output "dev-folders" {
|
# tftest skip (uses data sources)
|
||||||
value = module.my-dev.folders
|
```
|
||||||
|
|
||||||
|
### Projects under org with folder/project exclusions
|
||||||
|
```hcl
|
||||||
|
module "my-filtered" {
|
||||||
|
source = "./fabric/modules/projects-data-source"
|
||||||
|
parent = "organizations/123456789"
|
||||||
|
ignore_projects = [
|
||||||
|
"sandbox-*", # wildcard ignore
|
||||||
|
"project-full-id", # specific project id
|
||||||
|
"0123456789" # specific project number
|
||||||
|
]
|
||||||
|
|
||||||
|
include_projects = [
|
||||||
|
"sandbox-114", # include specific project which was excluded by wildcard
|
||||||
|
"415216609246" # include specific project which was excluded by wildcard (by project number)
|
||||||
|
]
|
||||||
|
|
||||||
|
ignore_folders = [ # subfolders are ingoner as well
|
||||||
|
"343991594985",
|
||||||
|
"437102807785",
|
||||||
|
"345245235245"
|
||||||
|
]
|
||||||
|
query = "state:ACTIVE"
|
||||||
|
}
|
||||||
|
|
||||||
|
output "filtered-projects" {
|
||||||
|
value = module.my-filtered.projects
|
||||||
}
|
}
|
||||||
|
|
||||||
# tftest skip (uses data sources)
|
# tftest skip (uses data sources)
|
||||||
|
|
||||||
```
|
```
|
||||||
<!-- BEGIN TFDOC -->
|
<!-- BEGIN TFDOC -->
|
||||||
|
|
||||||
|
@ -50,15 +79,17 @@ output "dev-folders" {
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [parent](variables.tf#L23) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | <code>string</code> | ✓ | |
|
| [parent](variables.tf#L55) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | <code>string</code> | ✓ | |
|
||||||
| [filter](variables.tf#L17) | A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters). | <code>string</code> | | <code>"lifecycleState:ACTIVE"</code> |
|
| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | <code>list(string)</code> | | <code>[]</code> |
|
||||||
|
| [ignore_projects](variables.tf#L28) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | <code>list(string)</code> | | <code>[]</code> |
|
||||||
|
| [include_projects](variables.tf#L41) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | <code>list(string)</code> | | <code>[]</code> |
|
||||||
|
| [query](variables.tf#L64) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | <code>string</code> | | <code>"state:ACTIVE"</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
| name | description | sensitive |
|
| name | description | sensitive |
|
||||||
|---|---|:---:|
|
|---|---|:---:|
|
||||||
| [folders](outputs.tf#L17) | Map of folders attributes keyed by folder id. | |
|
| [project_numbers](outputs.tf#L17) | List of project numbers. | |
|
||||||
| [project_numbers](outputs.tf#L22) | List of project numbers. | |
|
| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. | |
|
||||||
| [projects](outputs.tf#L27) | Map of projects attributes keyed by projects id. | |
|
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* Copyright 2022 Google LLC
|
* Copyright 2023 Google LLC
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -15,129 +15,27 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
folders_l1_map = { for item in data.google_folders.folders_l1.folders : item.name => item }
|
_ignore_folder_numbers = [for folder_id in var.ignore_folders : trimprefix(folder_id, "folders/")]
|
||||||
|
_ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers))
|
||||||
folders_l2_map = merge([
|
query = var.query != "" ? (
|
||||||
for _, v in data.google_folders.folders_l2 :
|
format("%s%s", var.query, local._ignore_folders_query)
|
||||||
{ for item in v.folders : item.name => item }
|
) : (
|
||||||
]...)
|
format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND "))
|
||||||
|
|
||||||
folders_l3_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l3 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l4_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l4 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l5_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l5 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l6_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l6 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l7_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l7 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l8_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l8 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l9_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l9 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
folders_l10_map = merge([
|
|
||||||
for _, v in data.google_folders.folders_l10 :
|
|
||||||
{ for item in v.folders : item.name => item }
|
|
||||||
]...)
|
|
||||||
|
|
||||||
all_folders = merge(
|
|
||||||
local.folders_l1_map,
|
|
||||||
local.folders_l2_map,
|
|
||||||
local.folders_l3_map,
|
|
||||||
local.folders_l4_map,
|
|
||||||
local.folders_l5_map,
|
|
||||||
local.folders_l6_map,
|
|
||||||
local.folders_l7_map,
|
|
||||||
local.folders_l8_map,
|
|
||||||
local.folders_l9_map,
|
|
||||||
local.folders_l10_map
|
|
||||||
)
|
)
|
||||||
|
|
||||||
parent_ids = toset(concat(
|
ignore_patterns = [for item in var.ignore_projects : "^${replace(item, "*", ".*")}$"]
|
||||||
[split("/", var.parent)[1]],
|
ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$"
|
||||||
[for k, _ in local.all_folders : split("/", k)[1]]
|
projects_after_ignore = [for item in data.google_cloud_asset_resources_search_all.projects.results : item if(
|
||||||
))
|
length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0
|
||||||
|
) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/"))
|
||||||
projects = merge([
|
]
|
||||||
for _, v in data.google_projects.projects :
|
|
||||||
{ for item in v.projects : item.project_id => item }
|
|
||||||
]...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# 10 datasources are used to cover 10 possible nested layers in GCP organization hirerarcy.
|
data "google_cloud_asset_resources_search_all" "projects" {
|
||||||
data "google_folders" "folders_l1" {
|
provider = google-beta
|
||||||
parent_id = var.parent
|
scope = var.parent
|
||||||
}
|
asset_types = [
|
||||||
|
"cloudresourcemanager.googleapis.com/Project"
|
||||||
data "google_folders" "folders_l2" {
|
]
|
||||||
for_each = local.folders_l1_map
|
query = local.query
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l3" {
|
|
||||||
for_each = local.folders_l2_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l4" {
|
|
||||||
for_each = local.folders_l3_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l5" {
|
|
||||||
for_each = local.folders_l4_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l6" {
|
|
||||||
for_each = local.folders_l5_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l7" {
|
|
||||||
for_each = local.folders_l6_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l8" {
|
|
||||||
for_each = local.folders_l7_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l9" {
|
|
||||||
for_each = local.folders_l8_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
data "google_folders" "folders_l10" {
|
|
||||||
for_each = local.folders_l9_map
|
|
||||||
parent_id = each.value.name
|
|
||||||
}
|
|
||||||
|
|
||||||
# Getting all projects parented by any of the folders in the tree including root prg/folder provided by `parent` variable.
|
|
||||||
data "google_projects" "projects" {
|
|
||||||
for_each = local.parent_ids
|
|
||||||
filter = "parent.id:${each.value} ${var.filter}"
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* Copyright 2022 Google LLC
|
* Copyright 2023 Google LLC
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -14,17 +14,12 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
output "folders" {
|
|
||||||
description = "Map of folders attributes keyed by folder id."
|
|
||||||
value = local.all_folders
|
|
||||||
}
|
|
||||||
|
|
||||||
output "project_numbers" {
|
output "project_numbers" {
|
||||||
description = "List of project numbers."
|
description = "List of project numbers."
|
||||||
value = [for _, v in local.projects : v.number]
|
value = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")]
|
||||||
}
|
}
|
||||||
|
|
||||||
output "projects" {
|
output "projects" {
|
||||||
description = "Map of projects attributes keyed by projects id."
|
description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format."
|
||||||
value = local.projects
|
value = local.projects_after_ignore
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
/**
|
/**
|
||||||
* Copyright 2022 Google LLC
|
* Copyright 2023 Google LLC
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -14,10 +14,42 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
variable "filter" {
|
variable "ignore_folders" {
|
||||||
description = "A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters)."
|
description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable."
|
||||||
type = string
|
type = list(string)
|
||||||
default = "lifecycleState:ACTIVE"
|
default = []
|
||||||
|
# example exlusing a folder
|
||||||
|
# ignore_folders = [
|
||||||
|
# "folders/0123456789",
|
||||||
|
# "2345678901"
|
||||||
|
# ]
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ignore_projects" {
|
||||||
|
description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output."
|
||||||
|
type = list(string)
|
||||||
|
default = []
|
||||||
|
# example
|
||||||
|
#ignore_projects = [
|
||||||
|
# "dev-proj-1",
|
||||||
|
# "uat-proj-2",
|
||||||
|
# "0123456789",
|
||||||
|
# "prd-proj-*"
|
||||||
|
#]
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "include_projects" {
|
||||||
|
description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries."
|
||||||
|
type = list(string)
|
||||||
|
default = []
|
||||||
|
# example excluding all the projects starting with "prf-" except "prd-123457"
|
||||||
|
#ignore_projects = [
|
||||||
|
# "prd-*"
|
||||||
|
#]
|
||||||
|
#include_projects = [
|
||||||
|
# "prd-123457",
|
||||||
|
# "0123456789"
|
||||||
|
#]
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "parent" {
|
variable "parent" {
|
||||||
|
@ -28,3 +60,9 @@ variable "parent" {
|
||||||
error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
|
error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "query" {
|
||||||
|
description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)."
|
||||||
|
type = string
|
||||||
|
default = "state:ACTIVE"
|
||||||
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# Copyright 2022 Google LLC
|
# Copyright 2023 Google LLC
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
|
|
Loading…
Reference in New Issue