Merge pull request #1584 from ehorning/ehorning/support-gcs-object-upload
add support for object upload to gcs module
This commit is contained in:
commit
927c04a6d4
|
@ -90,14 +90,31 @@ module "bucket-gcs-notification" {
|
||||||
}
|
}
|
||||||
# tftest modules=1 resources=4 inventory=notification.yaml
|
# tftest modules=1 resources=4 inventory=notification.yaml
|
||||||
```
|
```
|
||||||
<!-- BEGIN TFDOC -->
|
|
||||||
|
|
||||||
|
### Example with object upload
|
||||||
|
|
||||||
|
```hcl
|
||||||
|
module "bucket" {
|
||||||
|
source = "./fabric/modules/gcs"
|
||||||
|
project_id = "myproject"
|
||||||
|
name = "my-bucket"
|
||||||
|
objects_to_upload = {
|
||||||
|
sample-data = {
|
||||||
|
name = "example-file.csv"
|
||||||
|
source = "data/example-file.csv"
|
||||||
|
content_type = "text/csv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# tftest modules=1 resources=2 inventory=object-upload.yaml
|
||||||
|
```
|
||||||
|
<!-- BEGIN TFDOC -->
|
||||||
## Variables
|
## Variables
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [name](variables.tf#L116) | Bucket name suffix. | <code>string</code> | ✓ | |
|
| [name](variables.tf#L116) | Bucket name suffix. | <code>string</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L145) | Bucket project id. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L171) | Bucket project id. | <code>string</code> | ✓ | |
|
||||||
| [cors](variables.tf#L17) | CORS configuration for the bucket. Defaults to null. | <code title="object({ origin = optional(list(string)) method = optional(list(string)) response_header = optional(list(string)) max_age_seconds = optional(number) })">object({…})</code> | | <code>null</code> |
|
| [cors](variables.tf#L17) | CORS configuration for the bucket. Defaults to null. | <code title="object({ origin = optional(list(string)) method = optional(list(string)) response_header = optional(list(string)) max_age_seconds = optional(number) })">object({…})</code> | | <code>null</code> |
|
||||||
| [encryption_key](variables.tf#L28) | KMS key that will be used for encryption. | <code>string</code> | | <code>null</code> |
|
| [encryption_key](variables.tf#L28) | KMS key that will be used for encryption. | <code>string</code> | | <code>null</code> |
|
||||||
| [force_destroy](variables.tf#L34) | Optional map to set force destroy keyed by name, defaults to false. | <code>bool</code> | | <code>false</code> |
|
| [force_destroy](variables.tf#L34) | Optional map to set force destroy keyed by name, defaults to false. | <code>bool</code> | | <code>false</code> |
|
||||||
|
@ -107,12 +124,13 @@ module "bucket-gcs-notification" {
|
||||||
| [location](variables.tf#L101) | Bucket location. | <code>string</code> | | <code>"EU"</code> |
|
| [location](variables.tf#L101) | Bucket location. | <code>string</code> | | <code>"EU"</code> |
|
||||||
| [logging_config](variables.tf#L107) | Bucket logging configuration. | <code title="object({ log_bucket = string log_object_prefix = optional(string) })">object({…})</code> | | <code>null</code> |
|
| [logging_config](variables.tf#L107) | Bucket logging configuration. | <code title="object({ log_bucket = string log_object_prefix = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||||
| [notification_config](variables.tf#L121) | GCS Notification configuration. | <code title="object({ enabled = bool payload_format = string topic_name = string sa_email = string event_types = optional(list(string)) custom_attributes = optional(map(string)) object_name_prefix = optional(string) })">object({…})</code> | | <code>null</code> |
|
| [notification_config](variables.tf#L121) | GCS Notification configuration. | <code title="object({ enabled = bool payload_format = string topic_name = string sa_email = string event_types = optional(list(string)) custom_attributes = optional(map(string)) object_name_prefix = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||||
| [prefix](variables.tf#L135) | Optional prefix used to generate the bucket name. | <code>string</code> | | <code>null</code> |
|
| [objects_to_upload](variables.tf#L135) | Objects to be uploaded to bucket. | <code title="map(object({ name = string metadata = optional(map(string)) content = optional(string) source = optional(string) cache_control = optional(string) content_disposition = optional(string) content_encoding = optional(string) content_language = optional(string) content_type = optional(string) event_based_hold = optional(bool) temporary_hold = optional(bool) detect_md5hash = optional(string) storage_class = optional(string) kms_key_name = optional(string) customer_encryption = optional(object({ encryption_algorithm = optional(string) encryption_key = string })) }))">map(object({…}))</code> | | <code>{}</code> |
|
||||||
| [retention_policy](variables.tf#L150) | Bucket retention policy. | <code title="object({ retention_period = number is_locked = optional(bool) })">object({…})</code> | | <code>null</code> |
|
| [prefix](variables.tf#L161) | Optional prefix used to generate the bucket name. | <code>string</code> | | <code>null</code> |
|
||||||
| [storage_class](variables.tf#L159) | Bucket storage class. | <code>string</code> | | <code>"MULTI_REGIONAL"</code> |
|
| [retention_policy](variables.tf#L176) | Bucket retention policy. | <code title="object({ retention_period = number is_locked = optional(bool) })">object({…})</code> | | <code>null</code> |
|
||||||
| [uniform_bucket_level_access](variables.tf#L169) | Allow using object ACLs (false) or not (true, this is the recommended behavior) , defaults to true (which is the recommended practice, but not the behavior of storage API). | <code>bool</code> | | <code>true</code> |
|
| [storage_class](variables.tf#L185) | Bucket storage class. | <code>string</code> | | <code>"MULTI_REGIONAL"</code> |
|
||||||
| [versioning](variables.tf#L175) | Enable versioning, defaults to false. | <code>bool</code> | | <code>false</code> |
|
| [uniform_bucket_level_access](variables.tf#L195) | Allow using object ACLs (false) or not (true, this is the recommended behavior) , defaults to true (which is the recommended practice, but not the behavior of storage API). | <code>bool</code> | | <code>true</code> |
|
||||||
| [website](variables.tf#L181) | Bucket website. | <code title="object({ main_page_suffix = optional(string) not_found_page = optional(string) })">object({…})</code> | | <code>null</code> |
|
| [versioning](variables.tf#L201) | Enable versioning, defaults to false. | <code>bool</code> | | <code>false</code> |
|
||||||
|
| [website](variables.tf#L207) | Bucket website. | <code title="object({ main_page_suffix = optional(string) not_found_page = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
|
@ -122,7 +140,7 @@ module "bucket-gcs-notification" {
|
||||||
| [id](outputs.tf#L28) | Fully qualified bucket id. | |
|
| [id](outputs.tf#L28) | Fully qualified bucket id. | |
|
||||||
| [name](outputs.tf#L37) | Bucket name. | |
|
| [name](outputs.tf#L37) | Bucket name. | |
|
||||||
| [notification](outputs.tf#L46) | GCS Notification self link. | |
|
| [notification](outputs.tf#L46) | GCS Notification self link. | |
|
||||||
| [topic](outputs.tf#L51) | Topic ID used by GCS. | |
|
| [objects](outputs.tf#L51) | Objects in GCS bucket. | |
|
||||||
| [url](outputs.tf#L56) | Bucket URL. | |
|
| [topic](outputs.tf#L63) | Topic ID used by GCS. | |
|
||||||
|
| [url](outputs.tf#L68) | Bucket URL. | |
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -99,6 +99,35 @@ resource "google_storage_bucket" "bucket" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "google_storage_bucket_object" "objects" {
|
||||||
|
for_each = var.objects_to_upload
|
||||||
|
|
||||||
|
bucket = google_storage_bucket.bucket.id
|
||||||
|
name = each.value.name
|
||||||
|
metadata = each.value.metadata
|
||||||
|
content = each.value.content
|
||||||
|
source = each.value.source
|
||||||
|
cache_control = each.value.cache_control
|
||||||
|
content_disposition = each.value.content_disposition
|
||||||
|
content_encoding = each.value.content_encoding
|
||||||
|
content_language = each.value.content_language
|
||||||
|
content_type = each.value.content_type
|
||||||
|
event_based_hold = each.value.event_based_hold
|
||||||
|
temporary_hold = each.value.temporary_hold
|
||||||
|
detect_md5hash = each.value.detect_md5hash
|
||||||
|
storage_class = each.value.storage_class
|
||||||
|
kms_key_name = each.value.kms_key_name
|
||||||
|
|
||||||
|
dynamic "customer_encryption" {
|
||||||
|
for_each = each.value.customer_encryption == null ? [] : [""]
|
||||||
|
|
||||||
|
content {
|
||||||
|
encryption_algorithm = each.value.customer_encryption.encryption_algorithm
|
||||||
|
encryption_key = each.value.customer_encryption.encryption_key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
resource "google_storage_bucket_iam_binding" "bindings" {
|
resource "google_storage_bucket_iam_binding" "bindings" {
|
||||||
for_each = var.iam
|
for_each = var.iam
|
||||||
bucket = google_storage_bucket.bucket.name
|
bucket = google_storage_bucket.bucket.name
|
||||||
|
|
|
@ -48,6 +48,18 @@ output "notification" {
|
||||||
value = local.notification ? google_storage_notification.notification[0].self_link : null
|
value = local.notification ? google_storage_notification.notification[0].self_link : null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
output "objects" {
|
||||||
|
description = "Objects in GCS bucket."
|
||||||
|
value = { for k, v in google_storage_bucket_object.objects : k => {
|
||||||
|
crc32c = v.crc32c
|
||||||
|
md5hash = v.md5hash
|
||||||
|
self_link = v.self_link
|
||||||
|
output_name = v.output_name
|
||||||
|
media_link = v.media_link
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
output "topic" {
|
output "topic" {
|
||||||
description = "Topic ID used by GCS."
|
description = "Topic ID used by GCS."
|
||||||
value = local.notification ? google_pubsub_topic.topic[0].id : null
|
value = local.notification ? google_pubsub_topic.topic[0].id : null
|
||||||
|
|
|
@ -132,6 +132,32 @@ variable "notification_config" {
|
||||||
default = null
|
default = null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "objects_to_upload" {
|
||||||
|
description = "Objects to be uploaded to bucket."
|
||||||
|
type = map(object({
|
||||||
|
name = string
|
||||||
|
metadata = optional(map(string))
|
||||||
|
content = optional(string)
|
||||||
|
source = optional(string)
|
||||||
|
cache_control = optional(string)
|
||||||
|
content_disposition = optional(string)
|
||||||
|
content_encoding = optional(string)
|
||||||
|
content_language = optional(string)
|
||||||
|
content_type = optional(string)
|
||||||
|
event_based_hold = optional(bool)
|
||||||
|
temporary_hold = optional(bool)
|
||||||
|
detect_md5hash = optional(string)
|
||||||
|
storage_class = optional(string)
|
||||||
|
kms_key_name = optional(string)
|
||||||
|
customer_encryption = optional(object({
|
||||||
|
encryption_algorithm = optional(string)
|
||||||
|
encryption_key = string
|
||||||
|
}))
|
||||||
|
}))
|
||||||
|
default = {}
|
||||||
|
nullable = false
|
||||||
|
}
|
||||||
|
|
||||||
variable "prefix" {
|
variable "prefix" {
|
||||||
description = "Optional prefix used to generate the bucket name."
|
description = "Optional prefix used to generate the bucket name."
|
||||||
type = string
|
type = string
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
# Copyright 2023 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
values:
|
||||||
|
module.bucket.google_storage_bucket.bucket:
|
||||||
|
name: my-bucket
|
||||||
|
project: myproject
|
||||||
|
module.bucket.google_storage_bucket_object.objects["sample-data"]:
|
||||||
|
name: example-file.csv
|
||||||
|
source: data/example-file.csv
|
||||||
|
content_type: text/csv
|
||||||
|
|
||||||
|
counts:
|
||||||
|
google_storage_bucket: 1
|
||||||
|
google_storage_bucket_object: 1
|
Loading…
Reference in New Issue