From 39139e2fa19df5e42df781979c7c60bc24bb156c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 5 Mar 2024 13:13:02 +0100 Subject: [PATCH] add support for service account IAM variables to pf (#2130) --- modules/project-factory/README.md | 25 ++-- modules/project-factory/factory-projects.tf | 29 ++-- modules/project-factory/main.tf | 11 +- modules/project-factory/outputs.tf | 1 - modules/project-factory/variables.tf | 12 +- .../project_factory/examples/example.yaml | 130 +++++++++++++----- 6 files changed, 144 insertions(+), 64 deletions(-) diff --git a/modules/project-factory/README.md b/modules/project-factory/README.md index 97ffef1a..6a6983ac 100644 --- a/modules/project-factory/README.md +++ b/modules/project-factory/README.md @@ -43,18 +43,21 @@ Some examples on where to use each of the three sets are [provided below](#examp Service accounts can be managed as part of each project's YAML configuration. This allows creation of default service accounts used for GCE instances, in firewall rules, or for application-level credentials without resorting to a separate Terraform configuration. -Each service account is represented by one key and a set of optional key/value pairs in the `service_accounts` top-level YAML map, like in this example: +Each service account is represented by one key and a set of optional key/value pairs in the `service_accounts` top-level YAML map, which expose most of the variables available in the `iam-service-account` module: ```yaml service_accounts: be-0: {} fe-1: display_name: GCE frontend service account. + iam_self_roles: + - roles/storage.objectViewer iam_project_roles: - - roles/storage.objectViewer + my-host-project: + - roles/compute.networkUser ``` -Both the `display_name` and `iam_project_roles` attributes are optional. +Both the `display_name` and `iam_self_roles` attributes are optional. ### Billing budgets @@ -122,7 +125,7 @@ module "project-factory" { projects_data_path = "data/projects" } } -# tftest modules=8 resources=35 files=prj-app-1,prj-app-2,prj-app-3,budget-test-100 inventory=example.yaml +# tftest modules=8 resources=37 files=prj-app-1,prj-app-2,prj-app-3,budget-test-100 ``` ```yaml @@ -140,11 +143,17 @@ services: - storage.googleapis.com service_accounts: app-1-be: - iam_project_roles: + iam_self_roles: - roles/logging.logWriter - roles/monitoring.metricWriter + iam_project_roles: + my-host-project: + - roles/compute.networkUser app-1-fe: display_name: "Test app 1 frontend." + iam_project_roles: + my-host-project: + - roles/compute.networkUser billing_budgets: - test-100 # tftest-file id=prj-app-1 path=data/projects/prj-app-1.yaml @@ -223,9 +232,9 @@ update_rules: | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [factories_config](variables.tf#L91) | Path to folder with YAML resource description data files. | object({…}) | ✓ | | -| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | -| [data_merges](variables.tf#L49) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | -| [data_overrides](variables.tf#L69) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | +| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | object({…}) | | {} | +| [data_merges](variables.tf#L49) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | object({…}) | | {} | +| [data_overrides](variables.tf#L69) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | object({…}) | | {} | ## Outputs diff --git a/modules/project-factory/factory-projects.tf b/modules/project-factory/factory-projects.tf index d3c42a13..90e90a66 100644 --- a/modules/project-factory/factory-projects.tf +++ b/modules/project-factory/factory-projects.tf @@ -103,20 +103,31 @@ locals { var.data_defaults.tag_bindings ) # non-project resources - service_accounts = coalesce( - var.data_overrides.service_accounts, - try(v.service_accounts, null), - var.data_defaults.service_accounts - ) + service_accounts = try(v.service_accounts, {}) }) } service_accounts = flatten([ for k, v in local.projects : [ for name, opts in v.service_accounts : { - project = k - name = name - display_name = try(opts.display_name, "Terraform-managed.") - iam_project_roles = try(opts.iam_project_roles, null) + project = k + name = name + display_name = coalesce( + try(var.data_overrides.service_accounts.display_name, null), + try(opts.display_name, null), + try(var.data_defaults.service_accounts.display_name, null), + "Terraform-managed." + ) + iam_billing_roles = try(opts.iam_billing_roles, {}) + iam_organization_roles = try(opts.iam_organization_roles, {}) + iam_sa_roles = try(opts.iam_sa_roles, {}) + iam_project_roles = try(opts.iam_project_roles, {}) + iam_self_roles = distinct(concat( + try(var.data_overrides.service_accounts.iam_self_roles, []), + try(opts.iam_self_roles, []), + try(var.data_defaults.service_accounts.iam_self_roles, []), + )) + iam_storage_roles = try(opts.iam_storage_roles, {}) + opts = opts } ] ]) diff --git a/modules/project-factory/main.tf b/modules/project-factory/main.tf index 112ace7d..d01471b5 100644 --- a/modules/project-factory/main.tf +++ b/modules/project-factory/main.tf @@ -67,14 +67,17 @@ module "projects" { module "service-accounts" { source = "../iam-service-account" for_each = { - for k in local.service_accounts : "${k.project}-${k.name}" => k + for k in local.service_accounts : "${k.project}/${k.name}" => k } project_id = module.projects[each.value.project].project_id name = each.value.name display_name = each.value.display_name - iam_project_roles = each.value.iam_project_roles == null ? {} : { - (module.projects[each.value.project].project_id) = each.value.iam_project_roles - } + iam_project_roles = merge( + each.value.iam_project_roles, + each.value.iam_self_roles == null ? {} : { + (module.projects[each.value.project].project_id) = each.value.iam_self_roles + } + ) } module "billing-account" { diff --git a/modules/project-factory/outputs.tf b/modules/project-factory/outputs.tf index 99653a15..2c15ec99 100644 --- a/modules/project-factory/outputs.tf +++ b/modules/project-factory/outputs.tf @@ -21,7 +21,6 @@ output "projects" { output "service_accounts" { description = "Service account emails." - # TODO: group by project value = { for k, v in module.service-accounts : k => v.email } diff --git a/modules/project-factory/variables.tf b/modules/project-factory/variables.tf index 2e8c3338..a0c85a0e 100644 --- a/modules/project-factory/variables.tf +++ b/modules/project-factory/variables.tf @@ -38,8 +38,8 @@ variable "data_defaults" { tag_bindings = optional(map(string), {}) # non-project resources service_accounts = optional(map(object({ - display_name = optional(string, "Terraform-managed.") - iam_project_roles = optional(list(string)) + display_name = optional(string, "Terraform-managed.") + iam_self_roles = optional(list(string)) })), {}) }) nullable = false @@ -58,8 +58,8 @@ variable "data_merges" { tag_bindings = optional(map(string), {}) # non-project resources service_accounts = optional(map(object({ - display_name = optional(string, "Terraform-managed.") - iam_project_roles = optional(list(string)) + display_name = optional(string, "Terraform-managed.") + iam_self_roles = optional(list(string)) })), {}) }) nullable = false @@ -80,8 +80,8 @@ variable "data_overrides" { services = optional(list(string)) # non-project resources service_accounts = optional(map(object({ - display_name = optional(string, "Terraform-managed.") - iam_project_roles = optional(list(string)) + display_name = optional(string, "Terraform-managed.") + iam_self_roles = optional(list(string)) }))) }) nullable = false diff --git a/tests/modules/project_factory/examples/example.yaml b/tests/modules/project_factory/examples/example.yaml index cb0d2bda..1b3b50ad 100644 --- a/tests/modules/project_factory/examples/example.yaml +++ b/tests/modules/project_factory/examples/example.yaml @@ -13,6 +13,44 @@ # limitations under the License. values: + module.project-factory.module.billing-account[0].google_billing_budget.default["test-100"]: + all_updates_rule: + - disable_default_iam_recipients: true + pubsub_topic: null + schema_version: '1.0' + amount: + - last_period_amount: null + specified_amount: + - nanos: null + units: '100' + billing_account: 123456-123456-123456 + budget_filter: + - calendar_period: null + credit_types_treatment: INCLUDE_ALL_CREDITS + custom_period: [] + projects: + - projects/test-pf-prj-app-1 + resource_ancestors: + - folders/1234567890 + display_name: 100 dollars in current spend + threshold_rules: + - spend_basis: CURRENT_SPEND + threshold_percent: 0.5 + - spend_basis: CURRENT_SPEND + threshold_percent: 0.75 + timeouts: null + module.project-factory.module.billing-account[0].google_monitoring_notification_channel.default["billing-default"]: + description: null + display_name: Budget email notification billing-default. + enabled: true + force_delete: false + labels: + email_address: gcp-billing-admins@example.com + project: foo-billing-audit + sensitive_labels: [] + timeouts: null + type: email + user_labels: null module.project-factory.module.projects["prj-app-1"].data.google_storage_project_service_account.gcs_sa[0]: project: test-pf-prj-app-1 user_project: null @@ -74,6 +112,25 @@ values: host_project: foo-host service_project: test-pf-prj-app-2 timeouts: null + ? module.project-factory.module.projects["prj-app-2"].google_compute_subnetwork_iam_member.shared_vpc_host_robots["europe-west1:prod-default-ew1:cloudservices"] + : condition: [] + project: foo-host + region: europe-west1 + role: roles/compute.networkUser + subnetwork: prod-default-ew1 + ? module.project-factory.module.projects["prj-app-2"].google_compute_subnetwork_iam_member.shared_vpc_host_robots["europe-west1:prod-default-ew1:container-engine"] + : condition: [] + project: foo-host + region: europe-west1 + role: roles/compute.networkUser + subnetwork: prod-default-ew1 + ? module.project-factory.module.projects["prj-app-2"].google_compute_subnetwork_iam_member.shared_vpc_host_subnets_iam["europe-west1:prod-default-ew1:group:team-1@example.com"] + : condition: [] + member: group:team-1@example.com + project: foo-host + region: europe-west1 + role: roles/compute.networkUser + subnetwork: prod-default-ew1 module.project-factory.module.projects["prj-app-2"].google_essential_contacts_contact.contact["admin@example.com"]: email: admin@example.com language_tag: en @@ -81,6 +138,23 @@ values: - ALL parent: projects/test-pf-prj-app-2 timeouts: null + ? module.project-factory.module.projects["prj-app-2"].google_org_policy_policy.default["compute.restrictSharedVpcSubnetworks"] + : dry_run_spec: [] + name: projects/test-pf-prj-app-2/policies/compute.restrictSharedVpcSubnetworks + parent: projects/test-pf-prj-app-2 + spec: + - inherit_from_parent: null + reset: null + rules: + - allow_all: null + condition: [] + deny_all: null + enforce: null + values: + - allowed_values: + - projects/foo-host/regions/europe-west1/subnetworks/prod-default-ew1 + denied_values: null + timeouts: null module.project-factory.module.projects["prj-app-2"].google_project.project[0]: auto_create_network: false billing_account: 123456-123456-123456 @@ -110,18 +184,6 @@ values: : condition: [] project: foo-host role: roles/vpcaccess.user - ? module.project-factory.module.projects["prj-app-2"].google_compute_subnetwork_iam_member.shared_vpc_host_robots["europe-west1:prod-default-ew1:cloudservices"] - : condition: [ ] - project: foo-host - role: roles/compute.networkUser - ? module.project-factory.module.projects["prj-app-2"].google_compute_subnetwork_iam_member.shared_vpc_host_robots["europe-west1:prod-default-ew1:container-engine"] - : condition: [ ] - project: foo-host - role: roles/compute.networkUser - ? module.project-factory.module.projects["prj-app-2"].google_compute_subnetwork_iam_member.shared_vpc_host_subnets_iam["europe-west1:prod-default-ew1:group:team-1@example.com"] - : condition: [ ] - project: foo-host - role: roles/compute.networkUser module.project-factory.module.projects["prj-app-2"].google_project_service.project_services["compute.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false @@ -152,21 +214,6 @@ values: project: test-pf-prj-app-2 service: storage.googleapis.com timeouts: null - module.project-factory.module.projects["prj-app-2"].google_org_policy_policy.default["compute.restrictSharedVpcSubnetworks"]: - name: projects/test-pf-prj-app-2/policies/compute.restrictSharedVpcSubnetworks - parent: projects/test-pf-prj-app-2 - spec: - - inherit_from_parent: null - reset: null - rules: - - allow_all: null - condition: [ ] - deny_all: null - enforce: null - values: - - allowed_values: - - projects/foo-host/regions/europe-west1/subnetworks/prod-default-ew1 - denied_values: null module.project-factory.module.projects["prj-app-3"].data.google_storage_project_service_account.gcs_sa[0]: project: test-pf-prj-app-3 user_project: null @@ -210,33 +257,44 @@ values: project: test-pf-prj-app-3 service: storage.googleapis.com timeouts: null - ? module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_project_iam_member.project-roles["test-pf-prj-app-1-roles/logging.logWriter"] + ? module.project-factory.module.service-accounts["prj-app-1/app-1-be"].google_project_iam_member.project-roles["my-host-project-roles/compute.networkUser"] + : condition: [] + project: my-host-project + role: roles/compute.networkUser + ? module.project-factory.module.service-accounts["prj-app-1/app-1-be"].google_project_iam_member.project-roles["test-pf-prj-app-1-roles/logging.logWriter"] : condition: [] project: test-pf-prj-app-1 role: roles/logging.logWriter - ? module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_project_iam_member.project-roles["test-pf-prj-app-1-roles/monitoring.metricWriter"] + ? module.project-factory.module.service-accounts["prj-app-1/app-1-be"].google_project_iam_member.project-roles["test-pf-prj-app-1-roles/monitoring.metricWriter"] : condition: [] project: test-pf-prj-app-1 role: roles/monitoring.metricWriter - module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_service_account.service_account[0]: + module.project-factory.module.service-accounts["prj-app-1/app-1-be"].google_service_account.service_account[0]: account_id: app-1-be + create_ignore_already_exists: null description: null disabled: false - display_name: null + display_name: Terraform-managed. project: test-pf-prj-app-1 timeouts: null - module.project-factory.module.service-accounts["prj-app-1-app-1-fe"].google_service_account.service_account[0]: + ? module.project-factory.module.service-accounts["prj-app-1/app-1-fe"].google_project_iam_member.project-roles["my-host-project-roles/compute.networkUser"] + : condition: [] + project: my-host-project + role: roles/compute.networkUser + module.project-factory.module.service-accounts["prj-app-1/app-1-fe"].google_service_account.service_account[0]: account_id: app-1-fe + create_ignore_already_exists: null description: null disabled: false display_name: Test app 1 frontend. project: test-pf-prj-app-1 timeouts: null - module.project-factory.module.service-accounts["prj-app-2-app-2-be"].google_service_account.service_account[0]: + module.project-factory.module.service-accounts["prj-app-2/app-2-be"].google_service_account.service_account[0]: account_id: app-2-be + create_ignore_already_exists: null description: null disabled: false - display_name: null + display_name: Terraform-managed. project: test-pf-prj-app-2 timeouts: null @@ -249,11 +307,11 @@ counts: google_monitoring_notification_channel: 1 google_org_policy_policy: 1 google_project: 3 - google_project_iam_member: 4 + google_project_iam_member: 6 google_project_service: 11 google_service_account: 3 google_storage_project_service_account: 3 modules: 8 - resources: 35 + resources: 37 outputs: {}