diff --git a/fast/stages/2-security/README.md b/fast/stages/2-security/README.md index 5ff903f0..0f6302aa 100644 --- a/fast/stages/2-security/README.md +++ b/fast/stages/2-security/README.md @@ -295,16 +295,16 @@ Some references that might be useful in setting up this stage: | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 0-bootstrap | | [billing_account](variables.tf#L25) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | object({…}) | ✓ | | 0-bootstrap | | [folder_ids](variables.tf#L38) | Folder name => id mappings, the 'security' folder name must exist. | object({…}) | ✓ | | 1-resman | -| [organization](variables.tf#L97) | Organization details. | object({…}) | ✓ | | 0-bootstrap | -| [prefix](variables.tf#L113) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | -| [service_accounts](variables.tf#L124) | Automation service accounts that can assign the encrypt/decrypt roles on keys. | object({…}) | ✓ | | 1-resman | +| [organization](variables.tf#L98) | Organization details. | object({…}) | ✓ | | 0-bootstrap | +| [prefix](variables.tf#L114) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 0-bootstrap | +| [service_accounts](variables.tf#L125) | Automation service accounts that can assign the encrypt/decrypt roles on keys. | object({…}) | ✓ | | 1-resman | | [groups](variables.tf#L46) | Group names to grant organization-level permissions. | map(string) | | {…} | 0-bootstrap | -| [kms_keys](variables.tf#L61) | KMS keys to create, keyed by name. | map(object({…})) | | {} | | -| [outputs_location](variables.tf#L107) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [vpc_sc_access_levels](variables.tf#L135) | VPC SC access level definitions. | map(object({…})) | | {} | | -| [vpc_sc_egress_policies](variables.tf#L164) | VPC SC egress policy definitions. | map(object({…})) | | {} | | -| [vpc_sc_ingress_policies](variables.tf#L184) | VPC SC ingress policy definitions. | map(object({…})) | | {} | | -| [vpc_sc_perimeters](variables.tf#L205) | VPC SC regular perimeter definitions. | object({…}) | | {} | | +| [kms_keys](variables.tf#L61) | KMS keys to create, keyed by name. | map(object({…})) | | {} | | +| [outputs_location](variables.tf#L108) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [vpc_sc_access_levels](variables.tf#L136) | VPC SC access level definitions. | map(object({…})) | | {} | | +| [vpc_sc_egress_policies](variables.tf#L165) | VPC SC egress policy definitions. | map(object({…})) | | {} | | +| [vpc_sc_ingress_policies](variables.tf#L185) | VPC SC ingress policy definitions. | map(object({…})) | | {} | | +| [vpc_sc_perimeters](variables.tf#L206) | VPC SC regular perimeter definitions. | object({…}) | | {} | | ## Outputs diff --git a/fast/stages/2-security/variables.tf b/fast/stages/2-security/variables.tf index fa439c8c..39a3daf8 100644 --- a/fast/stages/2-security/variables.tf +++ b/fast/stages/2-security/variables.tf @@ -74,6 +74,7 @@ variable "kms_keys" { iam = optional(map(list(string)), {}) iam_bindings = optional(map(object({ members = list(string) + role = string condition = optional(object({ expression = string title = string diff --git a/modules/gcs/README.md b/modules/gcs/README.md index 993e0435..2c3a85e3 100644 --- a/modules/gcs/README.md +++ b/modules/gcs/README.md @@ -5,27 +5,55 @@ ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" - prefix = "test" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" versioning = true labels = { cost-center = "devops" } } -# tftest modules=1 resources=1 inventory=simple.yaml +# tftest modules=1 resources=1 inventory=simple.yaml e2e ``` ### Example with Cloud KMS ```hcl +module "project" { + source = "./fabric/modules/project" + name = var.project_id + project_create = false +} + +module "kms" { + source = "./fabric/modules/kms" + project_id = var.project_id + keyring = { + location = "europe" # location of the KMS must match location of the bucket + name = "test" + } + keys = { + bucket_key = { + iam_bindings = { + bucket_key_iam = { + members = ["serviceAccount:${module.project.service_accounts.robots.storage}"] + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + } + } + } + } +} + module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" - encryption_key = "my-encryption-key" + encryption_key = module.kms.keys.bucket_key.id + location = "EU" } -# tftest modules=1 resources=1 inventory=cmek.yaml + +# tftest skip e2e ``` ### Example with retention policy and logging @@ -33,7 +61,8 @@ module "bucket" { ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" retention_policy = { retention_period = 100 @@ -52,7 +81,8 @@ module "bucket" { ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" lifecycle_rules = { lr-0 = { @@ -66,26 +96,33 @@ module "bucket" { } } } -# tftest modules=1 resources=1 inventory=lifecycle.yaml +# tftest modules=1 resources=1 inventory=lifecycle.yaml e2e ``` ### Minimal example with GCS notifications ```hcl +module "project" { + source = "./fabric/modules/project" + name = var.project_id + project_create = false +} + module "bucket-gcs-notification" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" notification_config = { enabled = true payload_format = "JSON_API_V1" - sa_email = "service-@gs-project-accounts.iam.gserviceaccount.com" # GCS SA email must be passed or fetched from projects module. + sa_email = module.project.service_accounts.robots.storage topic_name = "gcs-notification-topic" event_types = ["OBJECT_FINALIZE"] custom_attributes = {} } } -# tftest modules=1 resources=4 inventory=notification.yaml +# tftest skip e2e ``` ### Example with object upload @@ -93,17 +130,18 @@ module "bucket-gcs-notification" { ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" objects_to_upload = { sample-data = { name = "example-file.csv" - source = "data/example-file.csv" + source = "assets/example-file.csv" content_type = "text/csv" } } } -# tftest modules=1 resources=2 inventory=object-upload.yaml +# tftest modules=1 resources=2 inventory=object-upload.yaml e2e ``` ### Examples of IAM @@ -111,24 +149,26 @@ module "bucket" { ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" iam = { - "roles/storage.admin" = ["group:storage@example.com"] + "roles/storage.admin" = ["group:${var.group_email}"] } } -# tftest modules=1 resources=2 inventory=iam-authoritative.yaml +# tftest modules=1 resources=2 inventory=iam-authoritative.yaml e2e ``` ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" iam_bindings = { storage-admin-with-delegated_roles = { role = "roles/storage.admin" - members = ["group:storage@example.com"] + members = ["group:${var.group_email}"] condition = { title = "delegated-role-grants" expression = format( @@ -144,18 +184,19 @@ module "bucket" { } } } -# tftest modules=1 resources=2 inventory=iam-bindings.yaml +# tftest modules=1 resources=2 inventory=iam-bindings.yaml e2e ``` ```hcl module "bucket" { source = "./fabric/modules/gcs" - project_id = "myproject" + project_id = var.project_id + prefix = var.prefix name = "my-bucket" iam_bindings_additive = { storage-admin-with-delegated_roles = { role = "roles/storage.admin" - member = "group:storage@example.com" + member = "group:${var.group_email}" condition = { title = "delegated-role-grants" expression = format( @@ -171,7 +212,7 @@ module "bucket" { } } } -# tftest modules=1 resources=2 inventory=iam-bindings-additive.yaml +# tftest modules=1 resources=2 inventory=iam-bindings-additive.yaml e2e ``` ## Variables diff --git a/modules/kms/README.md b/modules/kms/README.md index 1d08fce8..e11dd1af 100644 --- a/modules/kms/README.md +++ b/modules/kms/README.md @@ -120,14 +120,14 @@ module "kms" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [keyring](variables.tf#L64) | Keyring attributes. | object({…}) | ✓ | | -| [project_id](variables.tf#L113) | Project id where the keyring will be created. | string | ✓ | | +| [project_id](variables.tf#L114) | Project id where the keyring will be created. | string | ✓ | | | [iam](variables.tf#L17) | Keyring IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | | [iam_bindings_additive](variables.tf#L39) | Keyring individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | [import_job](variables.tf#L54) | Keyring import job attributes. | object({…}) | | null | | [keyring_create](variables.tf#L72) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | -| [keys](variables.tf#L78) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | -| [tag_bindings](variables.tf#L118) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | {} | +| [keys](variables.tf#L78) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L119) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | {} | ## Outputs diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf index c3291546..52de8389 100644 --- a/modules/kms/variables.tf +++ b/modules/kms/variables.tf @@ -90,6 +90,7 @@ variable "keys" { iam = optional(map(list(string)), {}) iam_bindings = optional(map(object({ members = list(string) + role = string condition = optional(object({ expression = string title = string diff --git a/tests/examples/conftest.py b/tests/examples/conftest.py index 014815c5..49570f2c 100644 --- a/tests/examples/conftest.py +++ b/tests/examples/conftest.py @@ -37,7 +37,7 @@ def get_tftest_directive(s): def pytest_generate_tests(metafunc, test_group='example', - filter_tests=lambda x: True): + filter_tests=lambda x: 'skip' not in x): """Find all README.md files and collect code examples tagged for testing.""" if test_group in metafunc.fixturenames: readmes = FABRIC_ROOT.glob('**/README.md') @@ -70,8 +70,7 @@ def pytest_generate_tests(metafunc, test_group='example', index += 1 code = child.children[0].children tftest_tag = get_tftest_directive(code) - if tftest_tag and ('skip' in tftest_tag or - not filter_tests(tftest_tag)): + if tftest_tag and not filter_tests(tftest_tag): continue if child.lang == 'hcl': path = module.relative_to(FABRIC_ROOT) diff --git a/tests/examples_e2e/setup_module/main.tf b/tests/examples_e2e/setup_module/main.tf index e781d779..40a7dcb5 100644 --- a/tests/examples_e2e/setup_module/main.tf +++ b/tests/examples_e2e/setup_module/main.tf @@ -14,6 +14,9 @@ locals { prefix = "${var.prefix}-${var.timestamp}${var.suffix}" + jit_services = [ + "storage.googleapis.com", # no permissions granted by default + ] services = [ # trimmed down list of services, to be extended as needed "apigee.googleapis.com", @@ -93,6 +96,15 @@ resource "google_kms_crypto_key" "key" { rotation_period = "100000s" } +resource "google_project_service_identity" "jit_si" { + for_each = toset(local.jit_services) + provider = google-beta + project = google_project.project.project_id + service = each.value + depends_on = [google_project_service.project_service] +} + + resource "local_file" "terraform_tfvars" { filename = "e2e_tests.tfvars" content = templatefile("e2e_tests.tfvars.tftpl", { diff --git a/tests/modules/gcs/assets/example-file.csv b/tests/modules/gcs/assets/example-file.csv new file mode 100644 index 00000000..33ae9af1 --- /dev/null +++ b/tests/modules/gcs/assets/example-file.csv @@ -0,0 +1 @@ +example,file diff --git a/tests/modules/gcs/examples/cmek.yaml b/tests/modules/gcs/examples/cmek.yaml index ee92a5d2..1f20a9ef 100644 --- a/tests/modules/gcs/examples/cmek.yaml +++ b/tests/modules/gcs/examples/cmek.yaml @@ -14,10 +14,10 @@ values: module.bucket.google_storage_bucket.bucket: - encryption: - - default_kms_key_name: my-encryption-key - name: my-bucket - project: myproject +# encryption: __missing__ +# - default_kms_key_name: + name: test-my-bucket + project: project-id counts: google_storage_bucket: 1 diff --git a/tests/modules/gcs/examples/iam-authoritative.yaml b/tests/modules/gcs/examples/iam-authoritative.yaml index 8956adc6..0be40cfb 100644 --- a/tests/modules/gcs/examples/iam-authoritative.yaml +++ b/tests/modules/gcs/examples/iam-authoritative.yaml @@ -24,8 +24,8 @@ values: lifecycle_rule: [] location: EU logging: [] - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id requester_pays: null retention_policy: [] storage_class: MULTI_REGIONAL @@ -36,10 +36,10 @@ values: autoclass: - enabled: false module.bucket.google_storage_bucket_iam_binding.authoritative["roles/storage.admin"]: - bucket: my-bucket + bucket: test-my-bucket condition: [] members: - - group:storage@example.com + - group:organization-admins@example.org role: roles/storage.admin counts: diff --git a/tests/modules/gcs/examples/iam-bindings-additive.yaml b/tests/modules/gcs/examples/iam-bindings-additive.yaml index 2c20f9aa..0c5f2311 100644 --- a/tests/modules/gcs/examples/iam-bindings-additive.yaml +++ b/tests/modules/gcs/examples/iam-bindings-additive.yaml @@ -24,8 +24,8 @@ values: lifecycle_rule: [] location: EU logging: [] - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id requester_pays: null retention_policy: [] storage_class: MULTI_REGIONAL @@ -36,12 +36,12 @@ values: autoclass: - enabled: false module.bucket.google_storage_bucket_iam_member.bindings["storage-admin-with-delegated_roles"]: - bucket: my-bucket + bucket: test-my-bucket condition: - description: null expression: api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly(['roles/storage.objectAdmin','roles/storage.objectViewer']) title: delegated-role-grants - member: group:storage@example.com + member: group:organization-admins@example.org role: roles/storage.admin counts: diff --git a/tests/modules/gcs/examples/iam-bindings.yaml b/tests/modules/gcs/examples/iam-bindings.yaml index 45113fae..3111f57b 100644 --- a/tests/modules/gcs/examples/iam-bindings.yaml +++ b/tests/modules/gcs/examples/iam-bindings.yaml @@ -24,8 +24,8 @@ values: lifecycle_rule: [] location: EU logging: [] - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id requester_pays: null retention_policy: [] storage_class: MULTI_REGIONAL @@ -36,13 +36,13 @@ values: autoclass: - enabled: false module.bucket.google_storage_bucket_iam_binding.bindings["storage-admin-with-delegated_roles"]: - bucket: my-bucket + bucket: test-my-bucket condition: - description: null expression: api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly(['roles/storage.objectAdmin','roles/storage.objectViewer']) title: delegated-role-grants members: - - group:storage@example.com + - group:organization-admins@example.org role: roles/storage.admin counts: diff --git a/tests/modules/gcs/examples/lifecycle.yaml b/tests/modules/gcs/examples/lifecycle.yaml index 69eeea41..221a4a21 100644 --- a/tests/modules/gcs/examples/lifecycle.yaml +++ b/tests/modules/gcs/examples/lifecycle.yaml @@ -29,8 +29,8 @@ values: matches_suffix: [] noncurrent_time_before: '' num_newer_versions: null - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id counts: google_storage_bucket: 1 diff --git a/tests/modules/gcs/examples/notification.yaml b/tests/modules/gcs/examples/notification.yaml index 9536e89b..81fde73e 100644 --- a/tests/modules/gcs/examples/notification.yaml +++ b/tests/modules/gcs/examples/notification.yaml @@ -16,10 +16,10 @@ values: module.bucket-gcs-notification.google_pubsub_topic.topic[0]: {} module.bucket-gcs-notification.google_pubsub_topic_iam_binding.binding[0]: {} module.bucket-gcs-notification.google_storage_bucket.bucket: - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id module.bucket-gcs-notification.google_storage_notification.notification[0]: - bucket: my-bucket + bucket: test-my-bucket event_types: - OBJECT_FINALIZE payload_format: JSON_API_V1 diff --git a/tests/modules/gcs/examples/object-upload.yaml b/tests/modules/gcs/examples/object-upload.yaml index 10e0ae10..8076e63b 100644 --- a/tests/modules/gcs/examples/object-upload.yaml +++ b/tests/modules/gcs/examples/object-upload.yaml @@ -14,11 +14,11 @@ values: module.bucket.google_storage_bucket.bucket: - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id module.bucket.google_storage_bucket_object.objects["sample-data"]: name: example-file.csv - source: data/example-file.csv + source: assets/example-file.csv content_type: text/csv counts: diff --git a/tests/modules/gcs/examples/retention-logging.yaml b/tests/modules/gcs/examples/retention-logging.yaml index 96241420..f92c1a6a 100644 --- a/tests/modules/gcs/examples/retention-logging.yaml +++ b/tests/modules/gcs/examples/retention-logging.yaml @@ -16,8 +16,8 @@ values: module.bucket.google_storage_bucket.bucket: logging: - log_bucket: log-bucket - name: my-bucket - project: myproject + name: test-my-bucket + project: project-id retention_policy: - is_locked: true retention_period: 100 diff --git a/tests/modules/gcs/examples/simple.yaml b/tests/modules/gcs/examples/simple.yaml index 0bc34c06..66120499 100644 --- a/tests/modules/gcs/examples/simple.yaml +++ b/tests/modules/gcs/examples/simple.yaml @@ -26,7 +26,7 @@ values: location: EU logging: [] name: test-my-bucket - project: myproject + project: project-id requester_pays: null retention_policy: [] storage_class: MULTI_REGIONAL