Net dash cfv2 (#1859)
* Handling SQL IP address issue * reverting one change * Improving this fix based on wiktor's feedback * formatting * Adding supporting for Cloud Function v2 (60 minutes timeout vs 9 minutes timeout) * Removing useless comment * formatting * updating inputs/outputs documentation * feedback from Julio * formatting * python formatting * formatting * formatting --------- Co-authored-by: Julio Castillo <jccb@google.com>
This commit is contained in:
parent
e1e8fe6566
commit
1f344b65e6
|
@ -59,32 +59,29 @@ A monitoring dashboard can be optionally be deployed int he same project by sett
|
||||||
dashboard_json_path = "../dashboards/quotas-utilization.json"
|
dashboard_json_path = "../dashboards/quotas-utilization.json"
|
||||||
```
|
```
|
||||||
<!-- BEGIN TFDOC -->
|
<!-- BEGIN TFDOC -->
|
||||||
|
|
||||||
## Variables
|
## Variables
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---:|:---:|:---:|
|
|---|---|:---:|:---:|:---:|
|
||||||
| [discovery_config](variables.tf#L48) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empty, every project under the discovery root node will be monitored. | <code title="object({ discovery_root = string monitored_folders = list(string) monitored_projects = list(string) custom_quota_file = optional(string) })">object({…})</code> | ✓ | |
|
| [discovery_config](variables.tf#L49) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empty, every project under the discovery root node will be monitored. | <code title="object({ discovery_root = string monitored_folders = list(string) monitored_projects = list(string) custom_quota_file = optional(string) })">object({…})</code> | ✓ | |
|
||||||
| [project_id](variables.tf#L100) | Project id where the Cloud Function will be deployed. | <code>string</code> | ✓ | |
|
| [project_id](variables.tf#L101) | Project id where the Cloud Function will be deployed. | <code>string</code> | ✓ | |
|
||||||
| [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | <code>string</code> | | <code>"./bundle.zip"</code> |
|
| [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | <code>string</code> | | <code>"./bundle.zip"</code> |
|
||||||
| [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | <code title="object({ bucket_name = optional(string) build_worker_pool_id = optional(string) bundle_path = optional(string, "./bundle.zip") debug = optional(bool, false) memory_mb = optional(number, 256) source_dir = optional(string, "../src") timeout_seconds = optional(number, 540) vpc_connector = optional(object({ name = string egress_settings = optional(string, "ALL_TRAFFIC") })) })">object({…})</code> | | <code>{}</code> |
|
| [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | <code title="object({ bucket_name = optional(string) build_worker_pool_id = optional(string) bundle_path = optional(string, "./bundle.zip") debug = optional(bool, false) memory_mb = optional(number, 256) source_dir = optional(string, "../src") timeout_seconds = optional(number, 540) version = optional(string, "v1") vpc_connector = optional(object({ name = string egress_settings = optional(string, "ALL_TRAFFIC") })) })">object({…})</code> | | <code>{}</code> |
|
||||||
| [dashboard_json_path](variables.tf#L42) | Optional monitoring dashboard to deploy. | <code>string</code> | | <code>null</code> |
|
| [dashboard_json_path](variables.tf#L43) | Optional monitoring dashboard to deploy. | <code>string</code> | | <code>null</code> |
|
||||||
| [grant_discovery_iam_roles](variables.tf#L66) | Optionally grant required IAM roles to Cloud Function service account. | <code>bool</code> | | <code>false</code> |
|
| [grant_discovery_iam_roles](variables.tf#L67) | Optionally grant required IAM roles to Cloud Function service account. | <code>bool</code> | | <code>false</code> |
|
||||||
| [labels](variables.tf#L73) | Billing labels used for the Cloud Function, and the project if project_create is true. | <code>map(string)</code> | | <code>{}</code> |
|
| [labels](variables.tf#L74) | Billing labels used for the Cloud Function, and the project if project_create is true. | <code>map(string)</code> | | <code>{}</code> |
|
||||||
| [monitoring_project](variables.tf#L79) | Project where generated metrics will be written. Default is to use the same project where the Cloud Function is deployed. | <code>string</code> | | <code>null</code> |
|
| [monitoring_project](variables.tf#L80) | Project where generated metrics will be written. Default is to use the same project where the Cloud Function is deployed. | <code>string</code> | | <code>null</code> |
|
||||||
| [name](variables.tf#L85) | Name used to create Cloud Function related resources. | <code>string</code> | | <code>"net-dash"</code> |
|
| [name](variables.tf#L86) | Name used to create Cloud Function related resources. | <code>string</code> | | <code>"net-dash"</code> |
|
||||||
| [project_create_config](variables.tf#L91) | Optional configuration if project creation is required. | <code title="object({ billing_account_id = string parent_id = optional(string) })">object({…})</code> | | <code>null</code> |
|
| [project_create_config](variables.tf#L92) | Optional configuration if project creation is required. | <code title="object({ billing_account_id = string parent_id = optional(string) })">object({…})</code> | | <code>null</code> |
|
||||||
| [region](variables.tf#L105) | Compute region where the Cloud Function will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
| [region](variables.tf#L106) | Compute region where the Cloud Function will be deployed. | <code>string</code> | | <code>"europe-west1"</code> |
|
||||||
| [schedule_config](variables.tf#L111) | Schedule timer configuration in crontab format. | <code>string</code> | | <code>"*/30 * * * *"</code> |
|
| [schedule_config](variables.tf#L112) | Schedule timer configuration in crontab format. | <code>string</code> | | <code>"*/30 * * * *"</code> |
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
| name | description | sensitive |
|
| name | description | sensitive |
|
||||||
|---|---|:---:|
|
|---|---|:---:|
|
||||||
| [bucket](outputs.tf#L17) | Cloud Function deployment bucket resource. | |
|
| [bucket](outputs.tf#L17) | Cloud Function deployment bucket resource. | |
|
||||||
| [cloud-function](outputs.tf#L22) | Cloud Function resource. | |
|
| [project_id](outputs.tf#L22) | Project id. | |
|
||||||
| [project_id](outputs.tf#L27) | Project id. | |
|
| [service_account](outputs.tf#L27) | Cloud Function service account. | |
|
||||||
| [service_account](outputs.tf#L32) | Cloud Function service account. | |
|
| [troubleshooting_payload](outputs.tf#L35) | Cloud Function payload used for manual triggering. | ✓ |
|
||||||
| [troubleshooting_payload](outputs.tf#L40) | Cloud Function payload used for manual triggering. | ✓ |
|
|
||||||
|
|
||||||
<!-- END TFDOC -->
|
<!-- END TFDOC -->
|
||||||
|
|
|
@ -16,6 +16,11 @@
|
||||||
|
|
||||||
locals {
|
locals {
|
||||||
discovery_roles = ["roles/compute.viewer", "roles/cloudasset.viewer"]
|
discovery_roles = ["roles/compute.viewer", "roles/cloudasset.viewer"]
|
||||||
|
function = (
|
||||||
|
var.cloud_function_config.version == "v1"
|
||||||
|
? module.cloud-function.0
|
||||||
|
: module.cloud-function-v2.0
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "random_string" "default" {
|
resource "random_string" "default" {
|
||||||
|
@ -38,11 +43,15 @@ module "project" {
|
||||||
"cloudfunctions.googleapis.com",
|
"cloudfunctions.googleapis.com",
|
||||||
"cloudscheduler.googleapis.com",
|
"cloudscheduler.googleapis.com",
|
||||||
"compute.googleapis.com",
|
"compute.googleapis.com",
|
||||||
"monitoring.googleapis.com"
|
"monitoring.googleapis.com",
|
||||||
|
"run.googleapis.com"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
### Cloud functions v1 ###
|
||||||
|
|
||||||
module "pubsub" {
|
module "pubsub" {
|
||||||
|
count = var.cloud_function_config.version == "v1" ? 1 : 0
|
||||||
source = "../../../../modules/pubsub"
|
source = "../../../../modules/pubsub"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = var.name
|
name = var.name
|
||||||
|
@ -51,6 +60,7 @@ module "pubsub" {
|
||||||
}
|
}
|
||||||
|
|
||||||
module "cloud-function" {
|
module "cloud-function" {
|
||||||
|
count = var.cloud_function_config.version == "v1" ? 1 : 0
|
||||||
source = "../../../../modules/cloud-function-v1"
|
source = "../../../../modules/cloud-function-v1"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
name = var.name
|
name = var.name
|
||||||
|
@ -77,7 +87,7 @@ module "cloud-function" {
|
||||||
service_account_create = true
|
service_account_create = true
|
||||||
trigger_config = {
|
trigger_config = {
|
||||||
event = "google.pubsub.topic.publish"
|
event = "google.pubsub.topic.publish"
|
||||||
resource = module.pubsub.topic.id
|
resource = module.pubsub[0].topic.id
|
||||||
}
|
}
|
||||||
vpc_connector = (
|
vpc_connector = (
|
||||||
var.cloud_function_config.vpc_connector == null
|
var.cloud_function_config.vpc_connector == null
|
||||||
|
@ -91,6 +101,7 @@ module "cloud-function" {
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_cloud_scheduler_job" "default" {
|
resource "google_cloud_scheduler_job" "default" {
|
||||||
|
count = var.cloud_function_config.version == "v1" ? 1 : 0
|
||||||
project = var.project_id
|
project = var.project_id
|
||||||
region = var.region
|
region = var.region
|
||||||
name = var.name
|
name = var.name
|
||||||
|
@ -99,7 +110,7 @@ resource "google_cloud_scheduler_job" "default" {
|
||||||
|
|
||||||
pubsub_target {
|
pubsub_target {
|
||||||
attributes = {}
|
attributes = {}
|
||||||
topic_name = module.pubsub.topic.id
|
topic_name = module.pubsub.0.topic.id
|
||||||
data = base64encode(jsonencode({
|
data = base64encode(jsonencode({
|
||||||
discovery_root = var.discovery_config.discovery_root
|
discovery_root = var.discovery_config.discovery_root
|
||||||
folders = var.discovery_config.monitored_folders
|
folders = var.discovery_config.monitored_folders
|
||||||
|
@ -118,6 +129,95 @@ resource "google_cloud_scheduler_job" "default" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
### Cloud functions v2 ###
|
||||||
|
|
||||||
|
module "cloud-function-v2" {
|
||||||
|
count = var.cloud_function_config.version == "v2" ? 1 : 0
|
||||||
|
source = "../../../../modules/cloud-function-v2"
|
||||||
|
project_id = module.project.project_id
|
||||||
|
name = var.name
|
||||||
|
bucket_name = coalesce(
|
||||||
|
var.cloud_function_config.bucket_name,
|
||||||
|
"${var.name}-${random_string.default.0.id}"
|
||||||
|
)
|
||||||
|
bucket_config = {
|
||||||
|
location = var.region
|
||||||
|
}
|
||||||
|
build_worker_pool = var.cloud_function_config.build_worker_pool_id
|
||||||
|
bundle_config = {
|
||||||
|
source_dir = var.cloud_function_config.source_dir
|
||||||
|
output_path = var.cloud_function_config.bundle_path
|
||||||
|
}
|
||||||
|
environment_variables = (
|
||||||
|
var.cloud_function_config.debug != true ? {} : { DEBUG = "1" }
|
||||||
|
)
|
||||||
|
function_config = {
|
||||||
|
entry_point = "main_cf_http"
|
||||||
|
memory_mb = var.cloud_function_config.memory_mb
|
||||||
|
timeout_seconds = var.cloud_function_config.timeout_seconds
|
||||||
|
}
|
||||||
|
service_account_create = true
|
||||||
|
vpc_connector = (
|
||||||
|
var.cloud_function_config.vpc_connector == null
|
||||||
|
? null
|
||||||
|
: {
|
||||||
|
create = false
|
||||||
|
name = var.cloud_function_config.vpc_connector.name
|
||||||
|
egress_settings = var.cloud_function_config.vpc_connector.egress_settings
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
module "cloud-scheduler-service-account" {
|
||||||
|
count = var.cloud_function_config.version == "v2" ? 1 : 0
|
||||||
|
source = "../../../../modules/iam-service-account"
|
||||||
|
project_id = module.project.project_id
|
||||||
|
name = "scheduler-sa"
|
||||||
|
iam_project_roles = {
|
||||||
|
"${module.project.project_id}" = [
|
||||||
|
"roles/run.invoker",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "google_cloud_scheduler_job" "scheduler-http" {
|
||||||
|
count = var.cloud_function_config.version == "v2" ? 1 : 0
|
||||||
|
project = var.project_id
|
||||||
|
region = var.region
|
||||||
|
name = var.name
|
||||||
|
schedule = var.schedule_config
|
||||||
|
time_zone = "UTC"
|
||||||
|
|
||||||
|
http_target {
|
||||||
|
http_method = "POST"
|
||||||
|
uri = module.cloud-function-v2.0.uri
|
||||||
|
body = base64encode(jsonencode({
|
||||||
|
discovery_root = var.discovery_config.discovery_root
|
||||||
|
folders = var.discovery_config.monitored_folders
|
||||||
|
projects = var.discovery_config.monitored_projects
|
||||||
|
monitoring_project = (
|
||||||
|
var.monitoring_project == null
|
||||||
|
? module.project.project_id
|
||||||
|
: var.monitoring_project
|
||||||
|
)
|
||||||
|
custom_quota = (
|
||||||
|
var.discovery_config.custom_quota_file == null
|
||||||
|
? { networks = {}, projects = {} }
|
||||||
|
: yamldecode(file(var.discovery_config.custom_quota_file))
|
||||||
|
)
|
||||||
|
}))
|
||||||
|
headers = {
|
||||||
|
"Content-Type" = "application/json"
|
||||||
|
}
|
||||||
|
oidc_token {
|
||||||
|
service_account_email = module.cloud-scheduler-service-account.0.email
|
||||||
|
audience = module.cloud-function-v2.0.uri
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### IAM configuration ###
|
||||||
|
|
||||||
resource "google_organization_iam_member" "discovery" {
|
resource "google_organization_iam_member" "discovery" {
|
||||||
for_each = toset(
|
for_each = toset(
|
||||||
var.grant_discovery_iam_roles &&
|
var.grant_discovery_iam_roles &&
|
||||||
|
@ -127,7 +227,7 @@ resource "google_organization_iam_member" "discovery" {
|
||||||
)
|
)
|
||||||
org_id = split("/", var.discovery_config.discovery_root)[1]
|
org_id = split("/", var.discovery_config.discovery_root)[1]
|
||||||
role = each.key
|
role = each.key
|
||||||
member = module.cloud-function.service_account_iam_email
|
member = var.cloud_function_config.version == "v1" ? module.cloud-function.0.service_account_iam_email : module.cloud-function-v2.0.service_account_iam_email
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_folder_iam_member" "discovery" {
|
resource "google_folder_iam_member" "discovery" {
|
||||||
|
@ -139,15 +239,16 @@ resource "google_folder_iam_member" "discovery" {
|
||||||
)
|
)
|
||||||
folder = var.discovery_config.discovery_root
|
folder = var.discovery_config.discovery_root
|
||||||
role = each.key
|
role = each.key
|
||||||
member = module.cloud-function.service_account_iam_email
|
member = var.cloud_function_config.version == "v1" ? module.cloud-function.0.service_account_iam_email : module.cloud-function-v2.0.service_account_iam_email
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "google_project_iam_member" "monitoring" {
|
resource "google_project_iam_member" "monitoring" {
|
||||||
project = module.project.project_id
|
project = module.project.project_id
|
||||||
role = "roles/monitoring.metricWriter"
|
role = "roles/monitoring.metricWriter"
|
||||||
member = module.cloud-function.service_account_iam_email
|
member = var.cloud_function_config.version == "v1" ? module.cloud-function.0.service_account_iam_email : module.cloud-function-v2.0.service_account_iam_email
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Importing default dashboard
|
||||||
resource "google_monitoring_dashboard" "dashboard" {
|
resource "google_monitoring_dashboard" "dashboard" {
|
||||||
count = var.dashboard_json_path == null ? 0 : 1
|
count = var.dashboard_json_path == null ? 0 : 1
|
||||||
project = var.project_id
|
project = var.project_id
|
||||||
|
|
|
@ -16,12 +16,7 @@
|
||||||
|
|
||||||
output "bucket" {
|
output "bucket" {
|
||||||
description = "Cloud Function deployment bucket resource."
|
description = "Cloud Function deployment bucket resource."
|
||||||
value = module.cloud-function.bucket
|
value = local.function.bucket
|
||||||
}
|
|
||||||
|
|
||||||
output "cloud-function" {
|
|
||||||
description = "Cloud Function resource."
|
|
||||||
value = module.cloud-function.function
|
|
||||||
}
|
}
|
||||||
|
|
||||||
output "project_id" {
|
output "project_id" {
|
||||||
|
@ -32,8 +27,8 @@ output "project_id" {
|
||||||
output "service_account" {
|
output "service_account" {
|
||||||
description = "Cloud Function service account."
|
description = "Cloud Function service account."
|
||||||
value = {
|
value = {
|
||||||
email = module.cloud-function.service_account_email
|
email = local.function.service_account_email
|
||||||
iam_email = module.cloud-function.service_account_iam_email
|
iam_email = local.function.service_account_iam_email
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,6 +36,6 @@ output "troubleshooting_payload" {
|
||||||
description = "Cloud Function payload used for manual triggering."
|
description = "Cloud Function payload used for manual triggering."
|
||||||
sensitive = true
|
sensitive = true
|
||||||
value = jsonencode({
|
value = jsonencode({
|
||||||
data = google_cloud_scheduler_job.default.pubsub_target.0.data
|
data = var.cloud_function_config.version == "v1" ? google_cloud_scheduler_job.default[0].pubsub_target.0.data : google_cloud_scheduler_job.scheduler-http[0].http_target.0.body
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,6 +30,7 @@ variable "cloud_function_config" {
|
||||||
memory_mb = optional(number, 256)
|
memory_mb = optional(number, 256)
|
||||||
source_dir = optional(string, "../src")
|
source_dir = optional(string, "../src")
|
||||||
timeout_seconds = optional(number, 540)
|
timeout_seconds = optional(number, 540)
|
||||||
|
version = optional(string, "v1")
|
||||||
vpc_connector = optional(object({
|
vpc_connector = optional(object({
|
||||||
name = string
|
name = string
|
||||||
egress_settings = optional(string, "ALL_TRAFFIC")
|
egress_settings = optional(string, "ALL_TRAFFIC")
|
||||||
|
|
|
@ -221,11 +221,11 @@ def main_cf_pubsub(event, context):
|
||||||
try:
|
try:
|
||||||
payload = json.loads(base64.b64decode(event['data']).decode('utf-8'))
|
payload = json.loads(base64.b64decode(event['data']).decode('utf-8'))
|
||||||
except (binascii.Error, json.JSONDecodeError) as e:
|
except (binascii.Error, json.JSONDecodeError) as e:
|
||||||
raise SystemExit(f'Invalid payload: e.args[0].')
|
raise SystemExit(f'Invalid payload: {e.args[0]}.')
|
||||||
discovery_root = payload.get('discovery_root')
|
discovery_root = payload.get('discovery_root')
|
||||||
monitoring_project = payload.get('monitoring_project')
|
monitoring_project = payload.get('monitoring_project')
|
||||||
if not discovery_root:
|
if not discovery_root:
|
||||||
LOGGER.critical('no discovery roo project specified')
|
LOGGER.critical('no discovery root project specified')
|
||||||
LOGGER.info(payload)
|
LOGGER.info(payload)
|
||||||
raise SystemExit(f'Invalid options')
|
raise SystemExit(f'Invalid options')
|
||||||
if not monitoring_project:
|
if not monitoring_project:
|
||||||
|
@ -249,6 +249,43 @@ def main_cf_pubsub(event, context):
|
||||||
do_timeseries(monitoring_project, timeseries, descriptors)
|
do_timeseries(monitoring_project, timeseries, descriptors)
|
||||||
|
|
||||||
|
|
||||||
|
def main_cf_http(request):
|
||||||
|
'Entry point for Cloud Function triggered by HTTP request.'
|
||||||
|
debug = os.environ.get('DEBUG')
|
||||||
|
logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
|
||||||
|
LOGGER.info('processing http payload')
|
||||||
|
try:
|
||||||
|
payload = json.loads(request.data)
|
||||||
|
except (binascii.Error, json.JSONDecodeError) as e:
|
||||||
|
raise SystemExit(f'Invalid payload: {e.args[0]}.')
|
||||||
|
discovery_root = payload.get('discovery_root')
|
||||||
|
monitoring_project = payload.get('monitoring_project')
|
||||||
|
if not discovery_root:
|
||||||
|
LOGGER.critical('no discovery root project specified')
|
||||||
|
LOGGER.info(payload)
|
||||||
|
raise SystemExit(f'Invalid options')
|
||||||
|
if not monitoring_project:
|
||||||
|
LOGGER.critical('no monitoring project specified')
|
||||||
|
LOGGER.info(payload)
|
||||||
|
raise SystemExit(f'Invalid options')
|
||||||
|
if discovery_root.partition('/')[0] not in ('folders', 'organizations'):
|
||||||
|
raise SystemExit(f'Invalid discovery root {discovery_root}.')
|
||||||
|
custom_quota = payload.get('custom_quota', {})
|
||||||
|
descriptors = []
|
||||||
|
folders = payload.get('folders', [])
|
||||||
|
projects = payload.get('projects', [])
|
||||||
|
resources = {}
|
||||||
|
timeseries = []
|
||||||
|
do_init(resources, discovery_root, monitoring_project, folders, projects,
|
||||||
|
custom_quota)
|
||||||
|
do_discovery(resources)
|
||||||
|
do_timeseries_calc(resources, descriptors, timeseries)
|
||||||
|
do_timeseries_descriptors(monitoring_project, resources['metric-descriptors'],
|
||||||
|
descriptors)
|
||||||
|
do_timeseries(monitoring_project, timeseries, descriptors)
|
||||||
|
return "Execution successful"
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
@click.option(
|
@click.option(
|
||||||
'--discovery-root', '-dr', required=True,
|
'--discovery-root', '-dr', required=True,
|
||||||
|
|
Loading…
Reference in New Issue