From 1f344b65e67ec71a22575297398d827222a4b23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 16 Nov 2023 15:45:44 +0100 Subject: [PATCH] Net dash cfv2 (#1859) * Handling SQL IP address issue * reverting one change * Improving this fix based on wiktor's feedback * formatting * Adding supporting for Cloud Function v2 (60 minutes timeout vs 9 minutes timeout) * Removing useless comment * formatting * updating inputs/outputs documentation * feedback from Julio * formatting * python formatting * formatting * formatting --------- Co-authored-by: Julio Castillo --- .../deploy-cloud-function/README.md | 31 +++-- .../deploy-cloud-function/main.tf | 113 +++++++++++++++++- .../deploy-cloud-function/outputs.tf | 13 +- .../deploy-cloud-function/variables.tf | 1 + .../network-dashboard/src/main.py | 41 ++++++- 5 files changed, 165 insertions(+), 34 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index acee13ee..1e917f88 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -59,32 +59,29 @@ A monitoring dashboard can be optionally be deployed int he same project by sett dashboard_json_path = "../dashboards/quotas-utilization.json" ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [discovery_config](variables.tf#L48) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empty, every project under the discovery root node will be monitored. | object({…}) | ✓ | | -| [project_id](variables.tf#L100) | Project id where the Cloud Function will be deployed. | string | ✓ | | +| [discovery_config](variables.tf#L49) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empty, every project under the discovery root node will be monitored. | object({…}) | ✓ | | +| [project_id](variables.tf#L101) | Project id where the Cloud Function will be deployed. | string | ✓ | | | [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | -| [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | object({…}) | | {} | -| [dashboard_json_path](variables.tf#L42) | Optional monitoring dashboard to deploy. | string | | null | -| [grant_discovery_iam_roles](variables.tf#L66) | Optionally grant required IAM roles to Cloud Function service account. | bool | | false | -| [labels](variables.tf#L73) | Billing labels used for the Cloud Function, and the project if project_create is true. | map(string) | | {} | -| [monitoring_project](variables.tf#L79) | Project where generated metrics will be written. Default is to use the same project where the Cloud Function is deployed. | string | | null | -| [name](variables.tf#L85) | Name used to create Cloud Function related resources. | string | | "net-dash" | -| [project_create_config](variables.tf#L91) | Optional configuration if project creation is required. | object({…}) | | null | -| [region](variables.tf#L105) | Compute region where the Cloud Function will be deployed. | string | | "europe-west1" | -| [schedule_config](variables.tf#L111) | Schedule timer configuration in crontab format. | string | | "*/30 * * * *" | +| [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | object({…}) | | {} | +| [dashboard_json_path](variables.tf#L43) | Optional monitoring dashboard to deploy. | string | | null | +| [grant_discovery_iam_roles](variables.tf#L67) | Optionally grant required IAM roles to Cloud Function service account. | bool | | false | +| [labels](variables.tf#L74) | Billing labels used for the Cloud Function, and the project if project_create is true. | map(string) | | {} | +| [monitoring_project](variables.tf#L80) | Project where generated metrics will be written. Default is to use the same project where the Cloud Function is deployed. | string | | null | +| [name](variables.tf#L86) | Name used to create Cloud Function related resources. | string | | "net-dash" | +| [project_create_config](variables.tf#L92) | Optional configuration if project creation is required. | object({…}) | | null | +| [region](variables.tf#L106) | Compute region where the Cloud Function will be deployed. | string | | "europe-west1" | +| [schedule_config](variables.tf#L112) | Schedule timer configuration in crontab format. | string | | "*/30 * * * *" | ## Outputs | name | description | sensitive | |---|---|:---:| | [bucket](outputs.tf#L17) | Cloud Function deployment bucket resource. | | -| [cloud-function](outputs.tf#L22) | Cloud Function resource. | | -| [project_id](outputs.tf#L27) | Project id. | | -| [service_account](outputs.tf#L32) | Cloud Function service account. | | -| [troubleshooting_payload](outputs.tf#L40) | Cloud Function payload used for manual triggering. | ✓ | - +| [project_id](outputs.tf#L22) | Project id. | | +| [service_account](outputs.tf#L27) | Cloud Function service account. | | +| [troubleshooting_payload](outputs.tf#L35) | Cloud Function payload used for manual triggering. | ✓ | diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf index 337c800d..dd59847b 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf @@ -16,6 +16,11 @@ locals { discovery_roles = ["roles/compute.viewer", "roles/cloudasset.viewer"] + function = ( + var.cloud_function_config.version == "v1" + ? module.cloud-function.0 + : module.cloud-function-v2.0 + ) } resource "random_string" "default" { @@ -38,11 +43,15 @@ module "project" { "cloudfunctions.googleapis.com", "cloudscheduler.googleapis.com", "compute.googleapis.com", - "monitoring.googleapis.com" + "monitoring.googleapis.com", + "run.googleapis.com" ] } +### Cloud functions v1 ### + module "pubsub" { + count = var.cloud_function_config.version == "v1" ? 1 : 0 source = "../../../../modules/pubsub" project_id = module.project.project_id name = var.name @@ -51,6 +60,7 @@ module "pubsub" { } module "cloud-function" { + count = var.cloud_function_config.version == "v1" ? 1 : 0 source = "../../../../modules/cloud-function-v1" project_id = module.project.project_id name = var.name @@ -77,7 +87,7 @@ module "cloud-function" { service_account_create = true trigger_config = { event = "google.pubsub.topic.publish" - resource = module.pubsub.topic.id + resource = module.pubsub[0].topic.id } vpc_connector = ( var.cloud_function_config.vpc_connector == null @@ -91,6 +101,7 @@ module "cloud-function" { } resource "google_cloud_scheduler_job" "default" { + count = var.cloud_function_config.version == "v1" ? 1 : 0 project = var.project_id region = var.region name = var.name @@ -99,7 +110,7 @@ resource "google_cloud_scheduler_job" "default" { pubsub_target { attributes = {} - topic_name = module.pubsub.topic.id + topic_name = module.pubsub.0.topic.id data = base64encode(jsonencode({ discovery_root = var.discovery_config.discovery_root folders = var.discovery_config.monitored_folders @@ -118,6 +129,95 @@ resource "google_cloud_scheduler_job" "default" { } } +### Cloud functions v2 ### + +module "cloud-function-v2" { + count = var.cloud_function_config.version == "v2" ? 1 : 0 + source = "../../../../modules/cloud-function-v2" + project_id = module.project.project_id + name = var.name + bucket_name = coalesce( + var.cloud_function_config.bucket_name, + "${var.name}-${random_string.default.0.id}" + ) + bucket_config = { + location = var.region + } + build_worker_pool = var.cloud_function_config.build_worker_pool_id + bundle_config = { + source_dir = var.cloud_function_config.source_dir + output_path = var.cloud_function_config.bundle_path + } + environment_variables = ( + var.cloud_function_config.debug != true ? {} : { DEBUG = "1" } + ) + function_config = { + entry_point = "main_cf_http" + memory_mb = var.cloud_function_config.memory_mb + timeout_seconds = var.cloud_function_config.timeout_seconds + } + service_account_create = true + vpc_connector = ( + var.cloud_function_config.vpc_connector == null + ? null + : { + create = false + name = var.cloud_function_config.vpc_connector.name + egress_settings = var.cloud_function_config.vpc_connector.egress_settings + } + ) +} + +module "cloud-scheduler-service-account" { + count = var.cloud_function_config.version == "v2" ? 1 : 0 + source = "../../../../modules/iam-service-account" + project_id = module.project.project_id + name = "scheduler-sa" + iam_project_roles = { + "${module.project.project_id}" = [ + "roles/run.invoker", + ] + } +} + +resource "google_cloud_scheduler_job" "scheduler-http" { + count = var.cloud_function_config.version == "v2" ? 1 : 0 + project = var.project_id + region = var.region + name = var.name + schedule = var.schedule_config + time_zone = "UTC" + + http_target { + http_method = "POST" + uri = module.cloud-function-v2.0.uri + body = base64encode(jsonencode({ + discovery_root = var.discovery_config.discovery_root + folders = var.discovery_config.monitored_folders + projects = var.discovery_config.monitored_projects + monitoring_project = ( + var.monitoring_project == null + ? module.project.project_id + : var.monitoring_project + ) + custom_quota = ( + var.discovery_config.custom_quota_file == null + ? { networks = {}, projects = {} } + : yamldecode(file(var.discovery_config.custom_quota_file)) + ) + })) + headers = { + "Content-Type" = "application/json" + } + oidc_token { + service_account_email = module.cloud-scheduler-service-account.0.email + audience = module.cloud-function-v2.0.uri + } + } +} + +### IAM configuration ### + resource "google_organization_iam_member" "discovery" { for_each = toset( var.grant_discovery_iam_roles && @@ -127,7 +227,7 @@ resource "google_organization_iam_member" "discovery" { ) org_id = split("/", var.discovery_config.discovery_root)[1] role = each.key - member = module.cloud-function.service_account_iam_email + member = var.cloud_function_config.version == "v1" ? module.cloud-function.0.service_account_iam_email : module.cloud-function-v2.0.service_account_iam_email } resource "google_folder_iam_member" "discovery" { @@ -139,15 +239,16 @@ resource "google_folder_iam_member" "discovery" { ) folder = var.discovery_config.discovery_root role = each.key - member = module.cloud-function.service_account_iam_email + member = var.cloud_function_config.version == "v1" ? module.cloud-function.0.service_account_iam_email : module.cloud-function-v2.0.service_account_iam_email } resource "google_project_iam_member" "monitoring" { project = module.project.project_id role = "roles/monitoring.metricWriter" - member = module.cloud-function.service_account_iam_email + member = var.cloud_function_config.version == "v1" ? module.cloud-function.0.service_account_iam_email : module.cloud-function-v2.0.service_account_iam_email } +# Importing default dashboard resource "google_monitoring_dashboard" "dashboard" { count = var.dashboard_json_path == null ? 0 : 1 project = var.project_id diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf index 0c2c50ab..0af90aca 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf @@ -16,12 +16,7 @@ output "bucket" { description = "Cloud Function deployment bucket resource." - value = module.cloud-function.bucket -} - -output "cloud-function" { - description = "Cloud Function resource." - value = module.cloud-function.function + value = local.function.bucket } output "project_id" { @@ -32,8 +27,8 @@ output "project_id" { output "service_account" { description = "Cloud Function service account." value = { - email = module.cloud-function.service_account_email - iam_email = module.cloud-function.service_account_iam_email + email = local.function.service_account_email + iam_email = local.function.service_account_iam_email } } @@ -41,6 +36,6 @@ output "troubleshooting_payload" { description = "Cloud Function payload used for manual triggering." sensitive = true value = jsonencode({ - data = google_cloud_scheduler_job.default.pubsub_target.0.data + data = var.cloud_function_config.version == "v1" ? google_cloud_scheduler_job.default[0].pubsub_target.0.data : google_cloud_scheduler_job.scheduler-http[0].http_target.0.body }) } diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf index 5793d8af..7beed1bf 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf @@ -30,6 +30,7 @@ variable "cloud_function_config" { memory_mb = optional(number, 256) source_dir = optional(string, "../src") timeout_seconds = optional(number, 540) + version = optional(string, "v1") vpc_connector = optional(object({ name = string egress_settings = optional(string, "ALL_TRAFFIC") diff --git a/blueprints/cloud-operations/network-dashboard/src/main.py b/blueprints/cloud-operations/network-dashboard/src/main.py index b05cac6a..e156ad89 100755 --- a/blueprints/cloud-operations/network-dashboard/src/main.py +++ b/blueprints/cloud-operations/network-dashboard/src/main.py @@ -221,11 +221,11 @@ def main_cf_pubsub(event, context): try: payload = json.loads(base64.b64decode(event['data']).decode('utf-8')) except (binascii.Error, json.JSONDecodeError) as e: - raise SystemExit(f'Invalid payload: e.args[0].') + raise SystemExit(f'Invalid payload: {e.args[0]}.') discovery_root = payload.get('discovery_root') monitoring_project = payload.get('monitoring_project') if not discovery_root: - LOGGER.critical('no discovery roo project specified') + LOGGER.critical('no discovery root project specified') LOGGER.info(payload) raise SystemExit(f'Invalid options') if not monitoring_project: @@ -249,6 +249,43 @@ def main_cf_pubsub(event, context): do_timeseries(monitoring_project, timeseries, descriptors) +def main_cf_http(request): + 'Entry point for Cloud Function triggered by HTTP request.' + debug = os.environ.get('DEBUG') + logging.basicConfig(level=logging.DEBUG if debug else logging.INFO) + LOGGER.info('processing http payload') + try: + payload = json.loads(request.data) + except (binascii.Error, json.JSONDecodeError) as e: + raise SystemExit(f'Invalid payload: {e.args[0]}.') + discovery_root = payload.get('discovery_root') + monitoring_project = payload.get('monitoring_project') + if not discovery_root: + LOGGER.critical('no discovery root project specified') + LOGGER.info(payload) + raise SystemExit(f'Invalid options') + if not monitoring_project: + LOGGER.critical('no monitoring project specified') + LOGGER.info(payload) + raise SystemExit(f'Invalid options') + if discovery_root.partition('/')[0] not in ('folders', 'organizations'): + raise SystemExit(f'Invalid discovery root {discovery_root}.') + custom_quota = payload.get('custom_quota', {}) + descriptors = [] + folders = payload.get('folders', []) + projects = payload.get('projects', []) + resources = {} + timeseries = [] + do_init(resources, discovery_root, monitoring_project, folders, projects, + custom_quota) + do_discovery(resources) + do_timeseries_calc(resources, descriptors, timeseries) + do_timeseries_descriptors(monitoring_project, resources['metric-descriptors'], + descriptors) + do_timeseries(monitoring_project, timeseries, descriptors) + return "Execution successful" + + @click.command() @click.option( '--discovery-root', '-dr', required=True,