From cdf53552d7b22d1f67590a40358ebdb9f5e0c2fc Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Mon, 3 Oct 2022 15:48:46 +0200 Subject: [PATCH 01/14] support for project level VPC firewall metrics --- .../network-dashboard/cloud-function/main.py | 32 ++-- .../cloud-function/metrics.yaml | 14 +- .../cloud-function/metrics/limits.py | 54 +++++++ .../cloud-function/metrics/metrics.py | 14 +- .../cloud-function/metrics/vpc_firewalls.py | 142 ++++++++++++++++++ 5 files changed, 239 insertions(+), 17 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index 9f284136..439b17c8 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -20,7 +20,7 @@ import time from google.cloud import monitoring_v3, asset_v1 from google.protobuf import field_mask_pb2 from googleapiclient import discovery -from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, subnets +from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, subnets, vpc_firewalls def get_monitored_projects_list(config): @@ -33,7 +33,7 @@ def get_monitored_projects_list(config): monitored_projects (List of strings): Full list of projects to be monitored ''' monitored_projects = config["monitored_projects"] - monitored_folders = os.environ.get("MONITORED_FOLDERS_LIST").split(",") + monitored_folders = [] #os.environ.get("MONITORED_FOLDERS_LIST").split(",") # Handling empty monitored folders list if monitored_folders == ['']: @@ -89,15 +89,18 @@ def monitoring_interval(): config = { # Organization ID containing the projects to be monitored - "organization": - os.environ.get("ORGANIZATION_ID"), + "organization": #os.environ.get("ORGANIZATION_ID"), + '34855741773', # list of projects from which function will get quotas information - "monitored_projects": - os.environ.get("MONITORED_PROJECTS_LIST").split(","), - "monitoring_project_link": - os.environ.get('MONITORING_PROJECT_ID'), - "monitoring_project_link": - f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", + "monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","), + [ + "mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0", + "mnoseda-dev-net-spoke-0" + ], + "monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'), + "monitoring-tlc", + "monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", + f"projects/monitoring-tlc", "monitoring_interval": monitoring_interval(), "limit_names": { @@ -143,6 +146,9 @@ def main(event, context): metrics_dict, limits_dict = metrics.create_metrics( config["monitoring_project_link"]) + project_quotas_dict = limits.get_quota_project_limit(config) + + firewalls_dict = vpc_firewalls.get_firewalls_dict(config) # IP utilization subnet level metrics subnets.get_subnets(config, metrics_dict) @@ -153,6 +159,10 @@ def main(event, context): l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") subnet_range_dict = networks.get_subnet_ranges_dict(config) + # Per Project metrics + vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, + firewalls_dict) + # Per Network metrics instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, limits_dict['number_of_instances_limit']) @@ -197,4 +207,4 @@ def main(event, context): if __name__ == "__main__": - main(None, None) \ No newline at end of file + main(None, None) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 05f5369c..be9cfe1f 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -159,4 +159,16 @@ metrics_per_peering_group: default_value: 300 utilization: name: dynamic_routes_per_peering_group_utilization - description: Number of Dynamic routes per peering group - utilization. \ No newline at end of file + description: Number of Dynamic routes per peering group - utilization. +metrics_per_project: + firewalls: + usage: + name: firewalls_per_project_vpc_usage + description: Number of VPC firewall rules in a project - usage. + limit: + #no default limit as we can assume quotas can be always read from gcloud API + name: firewalls_per_project_limit + description: Number of VPC firewall rules in a project - limit. + utilization: + name: firewalls_per_project_utilization + description: Number of VPC firewall rules in a project - utilization. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index d17f8be4..9178c6fe 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -19,6 +19,60 @@ from google.cloud import monitoring_v3 from . import metrics +def get_quotas_dict(quotas_list): + ''' + Creates a dictionary of quotas from a list, with lowe case keys + Parameters: + quotas_array (array): array of quotas + Returns: + quotas_dict (dict): dictionary of quotas + ''' + quota_keys = [q['metric'] for q in quotas_list] + quotas_dict = dict() + i = 0 + for key in quota_keys: + if ("metric" in quotas_list[i]): + del (quotas_list[i]["metric"]) + quotas_dict[key.lower()] = quotas_list[i] + i += 1 + return quotas_dict + + +def get_quota_project_limit(config, regions=["global"]): + ''' + Retrieves limit for a specific project quota + Parameters: + project_link (string): Project link. + Returns: + quotas (dict): quotas for all selected regions, default 'global' + ''' + try: + request = {} + quotas = dict() + for project in config["monitored_projects"]: + quotas[project] = dict() + if regions != ["global"]: + for region in regions: + request = config["clients"]["discovery_client"].compute.regions().get( + region=region, project=project) + response = request.execute() + quotas[project][region] = get_quotas_dict(response['quotas']) + else: + region = "global" + request = config["clients"]["discovery_client"].projects().get( + project=project, fields="quotas") + response = request.execute() + quotas[project][region] = get_quotas_dict(response['quotas']) + + return quotas + except exceptions.PermissionDenied as err: + print( + f"Warning: error reading quotas for {project}. " + + f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" + ) + return None + + def get_ppg(network_link, limit_dict): ''' Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 4194a13a..f6d8ac01 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -36,7 +36,9 @@ def create_metrics(monitoring_project): existing_metrics.append(desc.type) limits_dict = {} - with open("metrics.yaml", 'r') as stream: + with open( + "/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", + 'r') as stream: try: metrics_dict = yaml.safe_load(stream) @@ -52,8 +54,9 @@ def create_metrics(monitoring_project): # Subnet level metrics have a different limit: the subnet IP range size if sub_metric_key == "limit" and metric_name != "ip_usage_per_subnet": limits_dict_for_metric = {} - for network_link, limit_value in sub_metric["values"].items(): - limits_dict_for_metric[network_link] = limit_value + if "values" in sub_metric: + for network_link, limit_value in sub_metric["values"].items(): + limits_dict_for_metric[network_link] = limit_value limits_dict[sub_metric["name"]] = limits_dict_for_metric return metrics_dict, limits_dict @@ -84,7 +87,7 @@ def create_metric(metric_name, description, monitoring_project): def write_data_to_metric(config, monitored_project_id, value, metric_name, - network_name, subnet_id=None): + network_name=None, subnet_id=None): ''' Writes data to Cloud Monitoring custom metrics. Parameters: @@ -103,7 +106,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name, series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" - series.metric.labels["network_name"] = network_name + if network_name != None: + series.metric.labels["network_name"] = network_name series.metric.labels["project"] = monitored_project_id if subnet_id: series.metric.labels["subnet_id"] = subnet_id diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py new file mode 100644 index 00000000..cea8f751 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -0,0 +1,142 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from collections import defaultdict +from pydoc import doc +from collections import defaultdict +from google.protobuf import field_mask_pb2 +from . import metrics, networks, limits, peerings, routers + + +def get_firewalls_dict(config: dict): + ''' + Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization. + + Parameters: + config (dict): The dict containing config like clients and limits + Returns: + firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + ''' + + firewalls_dict = defaultdict(int) + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + response = config["clients"]["asset_client"].search_all_resources( + request={ + "scope": f"organizations/{config['organization']}", + "asset_types": ["compute.googleapis.com/Firewall"], + "read_mask": read_mask, + }) + for resource in response: + project_id = re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", + resource.name).group(2) + network_name = "" + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "network": + network_name = re.search("[a-z0-9\-]*$", field_value).group(0) + firewalls_dict[project_id] = defaultdict( + int + ) if not project_id in firewalls_dict else firewalls_dict[project_id] + firewalls_dict[project_id][ + network_name] = 1 if not network_name in firewalls_dict[ + project_id] else firewalls_dict[project_id][network_name] + 1 + break + break + return firewalls_dict + + +'''def get_firewalls_per_project_vpc(config, project_id ): + + Returns returns the VPC firewall rules defined in a project + + Parameters: + config (dict): The dict containing config like clients and limits + project_id (string): Project ID for the project containing the Cloud Router. + Returns: + sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project + + get_firewalls_dict(config) + sum_firewalls_per_vpc=dict() + sum_firewalls=0 + page_token=None + while (1): + request = config["clients"]["discovery_client"].firewalls().list( + project=project_id,pageToken=page_token) + response = request.execute() + sum_firewalls_per_vpc = dict() + if 'items' in response: + for firewall_rule in response['items']: + sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1 + else: + break + page_token = response['pageToken'] if 'pageToken' in response else None + if (not page_token): + break + + return sum_firewalls_per_vpc +''' + + +def get_firewalls_data(config, metrics_dict, project_quotas_dict, + firewalls_dict): + ''' + Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric. + + Parameters: + config (dict): The dict containing config like clients and limits + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. + firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + Returns: + None + ''' + for project in config["monitored_projects"]: + + current_quota_limit = project_quotas_dict[project]['global']["firewalls"] + if current_quota_limit is None: + print( + f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas" + ) + continue + + network_dict = networks.get_networks(config, project) + + project_usage = 0 + for net in network_dict: + usage = 0 + if net['network_name'] in firewalls_dict: + usage = firewalls_dict[project][net['network_name']] + project_usage += usage + metrics.write_data_to_metric( + config, project, usage, + metrics_dict["metrics_per_project"][f"firewalls"]["usage"]["name"], + net['network_name']) + + #firewall quotas are per project, not per single VPC + metrics.write_data_to_metric( + config, project, current_quota_limit['limit'], + metrics_dict["metrics_per_project"][f"firewalls"]["limit"]["name"]) + metrics.write_data_to_metric( + config, project, project_usage / current_quota_limit['limit'] + if current_quota_limit['limit'] != 0 else 0, + metrics_dict["metrics_per_project"][f"firewalls"]["utilization"] + ["name"]) + + print( + f"Wrote number of VPC Firewall Rules to metric for projects/{project}") From 7f2399bd703c9cfae8c2e29ecf129d40762f4783 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Tue, 4 Oct 2022 10:11:09 +0200 Subject: [PATCH 02/14] Added charts to dashboard, fixed a merge glitch, updated readme, removed hardcoded parameters --- .../network-dashboard/README.md | 3 +- .../network-dashboard/cloud-function/main.py | 19 ++- .../cloud-function/metrics/metrics.py | 4 +- .../dashboards/quotas-utilization.json | 131 +++++++++++------- 4 files changed, 96 insertions(+), 61 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index b300089b..254c8055 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -9,7 +9,7 @@ Here is an example of dashboard you can get with this solution: Here you see utilization (usage compared to the limit) for a specific metric (number of instances per VPC) for multiple VPCs and projects. -3 metrics are created: Usage, limit and utilization. You can follow each of these and create alerting policies if a threshold is reached. +Three metric descriptors are created for each monitored resource: usage, limit and utilization. You can follow each of these and create alerting policies if a threshold is reached. ## Usage @@ -45,6 +45,7 @@ The Cloud Function currently tracks usage, limit and utilization of: - Dynamic routes per VPC - Dynamic routes per VPC peering group - IP utilization per subnet (% of IP addresses used in a subnet) +- VPC firewall rules per project (VPC drill down is available for usage) It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index 439b17c8..7b36c79a 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -89,18 +89,15 @@ def monitoring_interval(): config = { # Organization ID containing the projects to be monitored - "organization": #os.environ.get("ORGANIZATION_ID"), - '34855741773', + "organization": + os.environ.get("ORGANIZATION_ID"), # list of projects from which function will get quotas information - "monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","), - [ - "mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0", - "mnoseda-dev-net-spoke-0" - ], - "monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'), - "monitoring-tlc", - "monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", - f"projects/monitoring-tlc", + "monitored_projects": + os.environ.get("MONITORED_PROJECTS_LIST").split(","), + "monitoring_project": + os.environ.get('MONITORING_PROJECT_ID'), + "monitoring_project_link": + f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", "monitoring_interval": monitoring_interval(), "limit_names": { diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index f6d8ac01..3bbca2e3 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -106,10 +106,10 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name, series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" + series.metric.labels["project"] = monitored_project_id if network_name != None: series.metric.labels["network_name"] = network_name - series.metric.labels["project"] = monitored_project_id - if subnet_id: + if subnet_id != None: series.metric.labels["subnet_id"] = subnet_id now = time.time() diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 6db2499a..8a70b38e 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -1,4 +1,5 @@ { + "category": "CUSTOM", "displayName": "quotas_utilization", "mosaicLayout": { "columns": 12, @@ -17,14 +18,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "1800s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -38,7 +42,9 @@ } } }, - "width": 6 + "width": 6, + "xPos": 0, + "yPos": 0 }, { "height": 4, @@ -54,14 +60,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -76,6 +85,7 @@ } }, "width": 6, + "xPos": 0, "yPos": 12 }, { @@ -92,14 +102,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -114,6 +127,7 @@ } }, "width": 6, + "xPos": 0, "yPos": 8 }, { @@ -130,14 +144,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -169,14 +186,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_INTERPOLATE" } } @@ -191,6 +211,7 @@ } }, "width": 6, + "xPos": 0, "yPos": 4 }, { @@ -207,14 +228,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -229,6 +253,7 @@ } }, "width": 6, + "xPos": 0, "yPos": 16 }, { @@ -245,14 +270,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -267,7 +295,8 @@ } }, "width": 6, - "xPos": 6 + "xPos": 6, + "yPos": 0 }, { "height": 4, @@ -283,14 +312,17 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -322,9 +354,11 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"" @@ -357,9 +391,11 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"" @@ -375,50 +411,53 @@ } }, "width": 6, - "yPos": 20 - }, - { - "height": 4, - "widget": { - "title": "ip_addresses_per_subnet_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, "xPos": 6, "yPos": 16 }, { "height": 4, "widget": { - "title": "dynamic_routes_ppg_utilization", + "title": "firewalls_per_project_usage", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": [ + "metric.label.\"project\"" + ], + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_usage\" resource.type=\"global\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 0, + "yPos": 20 + }, + { + "height": 4, + "widget": { + "title": "firewalls_per_project_utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -429,15 +468,14 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" + "crossSeriesReducer": "REDUCE_MAX", + "perSeriesAligner": "ALIGN_MAX" }, - "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_peering_group_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s" - } + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"" } } } @@ -454,6 +492,5 @@ "yPos": 20 } ] - }, - "name": "projects/347834224817/dashboards/1bdcd06a-030d-4977-bf4b-f32231aa3b77" + } } \ No newline at end of file From bebe9ed110879abb538b83b39ce17fb1c07b2fa8 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Wed, 5 Oct 2022 11:13:53 +0200 Subject: [PATCH 03/14] removed hardcoded debug value --- .../network-dashboard/cloud-function/metrics/metrics.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 3bbca2e3..155365b6 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -36,9 +36,7 @@ def create_metrics(monitoring_project): existing_metrics.append(desc.type) limits_dict = {} - with open( - "/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", - 'r') as stream: + with open("./metrics.yaml", 'r') as stream: try: metrics_dict = yaml.safe_load(stream) From 14da0eec2931983a9ea0f1269bcc44569ee83619 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Wed, 5 Oct 2022 12:27:35 +0200 Subject: [PATCH 04/14] removed obsolete code - compute api firewalls list --- .../cloud-function/metrics/vpc_firewalls.py | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py index cea8f751..1e00a8cd 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -61,38 +61,6 @@ def get_firewalls_dict(config: dict): return firewalls_dict -'''def get_firewalls_per_project_vpc(config, project_id ): - - Returns returns the VPC firewall rules defined in a project - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project containing the Cloud Router. - Returns: - sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project - - get_firewalls_dict(config) - sum_firewalls_per_vpc=dict() - sum_firewalls=0 - page_token=None - while (1): - request = config["clients"]["discovery_client"].firewalls().list( - project=project_id,pageToken=page_token) - response = request.execute() - sum_firewalls_per_vpc = dict() - if 'items' in response: - for firewall_rule in response['items']: - sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1 - else: - break - page_token = response['pageToken'] if 'pageToken' in response else None - if (not page_token): - break - - return sum_firewalls_per_vpc -''' - - def get_firewalls_data(config, metrics_dict, project_quotas_dict, firewalls_dict): ''' From c0573ce59693304adf75642535d55679ee0086b6 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 7 Oct 2022 11:34:00 +0200 Subject: [PATCH 05/14] fixed PR #856 comments --- .../network-dashboard/cloud-function/metrics.yaml | 2 +- .../network-dashboard/cloud-function/metrics/limits.py | 2 +- .../cloud-function/metrics/vpc_firewalls.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index be9cfe1f..2a9c8a4e 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -166,7 +166,7 @@ metrics_per_project: name: firewalls_per_project_vpc_usage description: Number of VPC firewall rules in a project - usage. limit: - #no default limit as we can assume quotas can be always read from gcloud API + # Firewalls limit is per project and we get the limit for the GCP quota API in vpc_firewalls.py name: firewalls_per_project_limit description: Number of VPC firewall rules in a project - limit. utilization: diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index 9178c6fe..a379a5c0 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -21,7 +21,7 @@ from . import metrics def get_quotas_dict(quotas_list): ''' - Creates a dictionary of quotas from a list, with lowe case keys + Creates a dictionary of quotas from a list, with lower case quota name as keys Parameters: quotas_array (array): array of quotas Returns: diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py index 1e00a8cd..8d15eacd 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -88,7 +88,8 @@ def get_firewalls_data(config, metrics_dict, project_quotas_dict, project_usage = 0 for net in network_dict: usage = 0 - if net['network_name'] in firewalls_dict: + if project in firewalls_dict and net['network_name'] in firewalls_dict[ + project]: usage = firewalls_dict[project][net['network_name']] project_usage += usage metrics.write_data_to_metric( @@ -96,7 +97,7 @@ def get_firewalls_data(config, metrics_dict, project_quotas_dict, metrics_dict["metrics_per_project"][f"firewalls"]["usage"]["name"], net['network_name']) - #firewall quotas are per project, not per single VPC + # firewall quotas are per project, not per single VPC metrics.write_data_to_metric( config, project, current_quota_limit['limit'], metrics_dict["metrics_per_project"][f"firewalls"]["limit"]["name"]) @@ -107,4 +108,4 @@ def get_firewalls_data(config, metrics_dict, project_quotas_dict, ["name"]) print( - f"Wrote number of VPC Firewall Rules to metric for projects/{project}") + f"Wrote number of VPC Firewall Rules to metric for projects/{project}") From 1f3cfe8e8e87e1deb3d8d5d924459128b26e3b39 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 7 Oct 2022 15:18:13 +0200 Subject: [PATCH 06/14] fixed dashboard comments --- .../dashboards/quotas-utilization.json | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 8a70b38e..5ba62af9 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -417,7 +417,7 @@ { "height": 4, "widget": { - "title": "firewalls_per_project_usage", + "title": "firewalls_per_project_vpc_usage", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -438,7 +438,7 @@ ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_usage\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_vpc_usage\" resource.type=\"global\"" } } } @@ -490,6 +490,43 @@ "width": 6, "xPos": 6, "yPos": 20 + }, + { + "height": 4, + "widget": { + "title": "ip_addresses_per_subnet_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 0, + "yPos": 24 } ] } From ac82a8ce7b7ab9f169483e517199e4e1c12b2940 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 7 Oct 2022 17:24:04 +0200 Subject: [PATCH 07/14] solved pul comments and fixed grouping on firewalls utilization chart --- .../dashboards/quotas-utilization.json | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 5ba62af9..923dbc84 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -22,13 +22,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "1800s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -64,13 +62,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -106,13 +102,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -148,13 +142,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -190,13 +182,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_INTERPOLATE" } } @@ -232,13 +222,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -274,13 +262,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -316,13 +302,11 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -358,7 +342,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"" @@ -395,7 +378,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"" @@ -425,7 +407,7 @@ "dataSets": [ { "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", + "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { "apiSource": "DEFAULT_CLOUD", @@ -438,11 +420,16 @@ ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_vpc_usage\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_vpc_usage\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } } } } ], + "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -473,13 +460,21 @@ "aggregation": { "alignmentPeriod": "60s", "crossSeriesReducer": "REDUCE_MAX", + "groupByFields": [ + "metric.label.\"project\"" + ], "perSeriesAligner": "ALIGN_MAX" }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } } } } ], + "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -509,7 +504,6 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" From 82f3daf917c20616b4239fd6a50c2d23663bc321 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Mon, 10 Oct 2022 15:52:54 +0200 Subject: [PATCH 08/14] Added Firewall Policies Monitoring, added buffered metric writes (CF now 4x faster on my laptop), switched to aligned timestamps (should avoid "sawtoothing") --- .../network-dashboard/cloud-function/main.py | 98 +++++++++++-------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index 7b36c79a..67f3316f 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -14,13 +14,14 @@ # limitations under the License. # +import re from distutils.command.config import config import os import time from google.cloud import monitoring_v3, asset_v1 from google.protobuf import field_mask_pb2 from googleapiclient import discovery -from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, subnets, vpc_firewalls +from metrics import ilb_fwrules, firewall_policies, instances, networks, metrics, limits, peerings, routes, subnets, vpc_firewalls def get_monitored_projects_list(config): @@ -119,6 +120,7 @@ config = { "asset_client": asset_v1.AssetServiceClient(), "monitoring_client": monitoring_v3.MetricServiceClient() }, + "series_buffer": [] } @@ -146,6 +148,7 @@ def main(event, context): project_quotas_dict = limits.get_quota_project_limit(config) firewalls_dict = vpc_firewalls.get_firewalls_dict(config) + firewall_policies_dict = firewall_policies.get_firewall_policies_dict(config) # IP utilization subnet level metrics subnets.get_subnets(config, metrics_dict) @@ -156,51 +159,60 @@ def main(event, context): l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") subnet_range_dict = networks.get_subnet_ranges_dict(config) - # Per Project metrics - vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, - firewalls_dict) + try: - # Per Network metrics - instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, - limits_dict['number_of_instances_limit']) - ilb_fwrules.get_forwarding_rules_data( - config, metrics_dict, l4_forwarding_rules_dict, - limits_dict['internal_forwarding_rules_l4_limit'], "L4") - ilb_fwrules.get_forwarding_rules_data( - config, metrics_dict, l7_forwarding_rules_dict, - limits_dict['internal_forwarding_rules_l7_limit'], "L7") - peerings.get_vpc_peering_data(config, metrics_dict, - limits_dict['number_of_vpc_peerings_limit']) - dynamic_routes_dict = routes.get_dynamic_routes( - config, metrics_dict, limits_dict['dynamic_routes_per_network_limit']) + # Per Project metrics + vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, + firewalls_dict) + # Per Firewall Policy metrics + firewall_policies.get_firewal_policies_data(config, metrics_dict, + firewall_policies_dict) + # Per Network metrics + instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, + limits_dict['number_of_instances_limit']) + ilb_fwrules.get_forwarding_rules_data( + config, metrics_dict, l4_forwarding_rules_dict, + limits_dict['internal_forwarding_rules_l4_limit'], "L4") + ilb_fwrules.get_forwarding_rules_data( + config, metrics_dict, l7_forwarding_rules_dict, + limits_dict['internal_forwarding_rules_l7_limit'], "L7") + peerings.get_vpc_peering_data(config, metrics_dict, + limits_dict['number_of_vpc_peerings_limit']) + dynamic_routes_dict = routes.get_dynamic_routes( + config, metrics_dict, limits_dict['dynamic_routes_per_network_limit']) - # Per VPC peering group metrics - metrics.get_pgg_data( - config, - metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], - gce_instance_dict, config["limit_names"]["GCE_INSTANCES"], - limits_dict['number_of_instances_ppg_limit']) - metrics.get_pgg_data( - config, metrics_dict["metrics_per_peering_group"] - ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict, - config["limit_names"]["L4"], - limits_dict['internal_forwarding_rules_l4_ppg_limit']) - metrics.get_pgg_data( - config, metrics_dict["metrics_per_peering_group"] - ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict, - config["limit_names"]["L7"], - limits_dict['internal_forwarding_rules_l7_ppg_limit']) - metrics.get_pgg_data( - config, metrics_dict["metrics_per_peering_group"] - ["subnet_ranges_per_peering_group"], subnet_range_dict, - config["limit_names"]["SUBNET_RANGES"], - limits_dict['number_of_subnet_IP_ranges_ppg_limit']) - routes.get_dynamic_routes_ppg( - config, metrics_dict["metrics_per_peering_group"] - ["dynamic_routes_per_peering_group"], dynamic_routes_dict, - limits_dict['dynamic_routes_per_peering_group_limit']) + # Per VPC peering group metrics + metrics.get_pgg_data( + config, + metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], + gce_instance_dict, config["limit_names"]["GCE_INSTANCES"], + limits_dict['number_of_instances_ppg_limit']) + metrics.get_pgg_data( + config, metrics_dict["metrics_per_peering_group"] + ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict, + config["limit_names"]["L4"], + limits_dict['internal_forwarding_rules_l4_ppg_limit']) + metrics.get_pgg_data( + config, metrics_dict["metrics_per_peering_group"] + ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict, + config["limit_names"]["L7"], + limits_dict['internal_forwarding_rules_l7_ppg_limit']) + metrics.get_pgg_data( + config, metrics_dict["metrics_per_peering_group"] + ["subnet_ranges_per_peering_group"], subnet_range_dict, + config["limit_names"]["SUBNET_RANGES"], + limits_dict['number_of_subnet_IP_ranges_ppg_limit']) + routes.get_dynamic_routes_ppg( + config, metrics_dict["metrics_per_peering_group"] + ["dynamic_routes_per_peering_group"], dynamic_routes_dict, + limits_dict['dynamic_routes_per_peering_group_limit']) + except Exception as e: + print("Error writing metrics") + print(e) + finally: + metrics.flush_series_buffer(config) - return 'Function executed successfully' + return 'Function execution completed' if __name__ == "__main__": From c52b623857c11636bf761359e6288f3dc40862b8 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Mon, 10 Oct 2022 15:53:14 +0200 Subject: [PATCH 09/14] Added Firewall Policies Monitoring Added buffering for writes (execution now 4x faster on my laptop) Added aligned timestamps per monitored resource type --- .../cloud-function/metrics.yaml | 14 +++ .../metrics/firewall_policies.py | 117 ++++++++++++++++++ .../cloud-function/metrics/ilb_fwrules.py | 40 +++--- .../cloud-function/metrics/instances.py | 39 +++--- .../cloud-function/metrics/limits.py | 25 ++-- .../cloud-function/metrics/metrics.py | 76 +++++++----- .../cloud-function/metrics/peerings.py | 71 ++++++----- .../cloud-function/metrics/routes.py | 47 +++---- .../cloud-function/metrics/subnets.py | 34 +++-- .../cloud-function/metrics/vpc_firewalls.py | 56 +++++---- 10 files changed, 365 insertions(+), 154 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 2a9c8a4e..b87dc1c3 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -172,3 +172,17 @@ metrics_per_project: utilization: name: firewalls_per_project_utilization description: Number of VPC firewall rules in a project - utilization. +metrics_per_firewall_policy: + firewall_policy_tuples: + usage: + name: firewall_policy_tuples_per_policy_usage + description: Number of tuples in a firewall policy - usage. + limit: + # This limit is not visibile through Google APIs, set default_value + name: firewall_policy_tuples_per_policy_limit + description: Number of tuples in a firewall policy - limit. + values: + default_value: 2000 + utilization: + name: firewall_policy_tuples_per_policy_utilization + description: Number of tuples in a firewall policy - utilization. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py new file mode 100644 index 00000000..8c9e33a2 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py @@ -0,0 +1,117 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from ast import AnnAssign + +import re +import time + +from collections import defaultdict +from pydoc import doc +from collections import defaultdict +from google.protobuf import field_mask_pb2 +from . import metrics, networks, limits + + +def get_firewall_policies_dict(config: dict): + ''' + Calls the Asset Inventory API to get all Firewall Policies under the GCP organization + + Parameters: + config (dict): The dict containing config like clients and limits + Returns: + firewal_policies_dict (dictionary of dictionary): Keys are policy ids, subkeys are policy field values + ''' + + firewall_policies_dict = defaultdict(int) + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + response = config["clients"]["asset_client"].search_all_resources( + request={ + "scope": f"organizations/{config['organization']}", + "asset_types": ["compute.googleapis.com/FirewallPolicy"], + "read_mask": read_mask, + }) + for resource in response: + for versioned in resource.versioned_resources: + firewall_policy = dict() + for field_name, field_value in versioned.resource.items(): + firewall_policy[field_name] = field_value + firewall_policies_dict[firewall_policy['id']] = firewall_policy + return firewall_policies_dict + + +def get_firewal_policies_data(config, metrics_dict, firewall_policies_dict): + ''' + Gets the data for VPC Firewall lorem ipsum + + Parameters: + config (dict): The dict containing config like clients and limits + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. + firewall_policies_dict (dictionary of of dictionary of string: string): Keys are policies ids, subkeys are policies values + Returns: + None + ''' + + current_tuples_limit = None + try: + current_tuples_limit = metrics_dict["metrics_per_firewall_policy"][ + "firewall_policy_tuples"]["limit"]["values"]["default_value"] + except Exception: + print( + f"Could not determine number of tuples metric limit due to missing default value" + ) + if current_tuples_limit < 0: + print( + f"Could not determine number of tuples metric limit as default value is <= 0" + ) + + timestamp = time.time() + for firewall_policy_key in firewall_policies_dict: + firewall_policy = firewall_policies_dict[firewall_policy_key] + + # may either be a org, a folder, or a project + # folder and org require to split {folder,organization}\/\w+ + parent = re.search("(\w+$)", firewall_policy["parent"]).group( + 1) if "parent" in firewall_policy else re.search( + "([\d,a-z,-]+)(\/[\d,a-z,-]+\/firewallPolicies/[\d,a-z,-]*$)", + firewall_policy["selfLink"]).group(1) + parent_type = re.search("(^\w+)", firewall_policy["parent"]).group( + 1) if "parent" in firewall_policy else "projects" + + metric_labels = {'parent': parent, 'parent_type': parent_type} + + metric_labels["name"] = firewall_policy[ + "displayName"] if "displayName" in firewall_policy else firewall_policy[ + "name"] + + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_firewall_policy"] + [f"firewall_policy_tuples"]["usage"]["name"], + firewall_policy['ruleTupleCount'], metric_labels, timestamp=timestamp) + if not current_tuples_limit == None and current_tuples_limit > 0: + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_firewall_policy"] + [f"firewall_policy_tuples"]["limit"]["name"], current_tuples_limit, + metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_firewall_policy"] + [f"firewall_policy_tuples"]["utilization"]["name"], + firewall_policy['ruleTupleCount'] / current_tuples_limit, + metric_labels, timestamp=timestamp) + + print(f"Buffered number tuples per Firewall Policy") diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py index 55490544..9b411b15 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py @@ -14,6 +14,8 @@ # limitations under the License. # +import time + from collections import defaultdict from google.protobuf import field_mask_pb2 from . import metrics, networks, limits @@ -75,15 +77,17 @@ def get_forwarding_rules_data(config, metrics_dict, forwarding_rules_dict, Returns: None ''' - for project in config["monitored_projects"]: - network_dict = networks.get_networks(config, project) + + timestamp = time.time() + for project_id in config["monitored_projects"]: + network_dict = networks.get_networks(config, project_id) current_quota_limit = limits.get_quota_current_limit( - config, f"projects/{project}", config["limit_names"][layer]) + config, f"projects/{project_id}", config["limit_names"][layer]) if current_quota_limit is None: print( - f"Could not write {layer} forwarding rules to metric for projects/{project} due to missing quotas" + f"Could not determine {layer} forwarding rules to metric for projects/{project_id} due to missing quotas" ) continue @@ -95,20 +99,24 @@ def get_forwarding_rules_data(config, metrics_dict, forwarding_rules_dict, usage = 0 if net['self_link'] in forwarding_rules_dict: usage = forwarding_rules_dict[net['self_link']] - metrics.write_data_to_metric( - config, project, usage, metrics_dict["metrics_per_network"] + + metric_labels = { + 'project': project_id, + 'network_name': net['network_name'] + } + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] [f"{layer.lower()}_forwarding_rules_per_network"]["usage"]["name"], - net['network_name']) - metrics.write_data_to_metric( - config, project, net['limit'], metrics_dict["metrics_per_network"] + usage, metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] [f"{layer.lower()}_forwarding_rules_per_network"]["limit"]["name"], - net['network_name']) - metrics.write_data_to_metric( - config, project, usage / net['limit'], - metrics_dict["metrics_per_network"] + net['limit'], metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] [f"{layer.lower()}_forwarding_rules_per_network"]["utilization"] - ["name"], net['network_name']) + ["name"], usage / net['limit'], metric_labels, timestamp=timestamp) print( - f"Wrote number of {layer} forwarding rules to metric for projects/{project}" - ) + f"Buffered number of {layer} forwarding rules to metric for projects/{project_id}" + ) \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py index ef78e67d..dda94c5a 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py @@ -14,6 +14,8 @@ # limitations under the License. # +import time + from code import interact from collections import defaultdict from . import metrics, networks, limits @@ -61,15 +63,16 @@ def get_gce_instances_data(config, metrics_dict, gce_instance_dict, limit_dict): Returns: gce_instance_dict ''' - - for project in config["monitored_projects"]: - network_dict = networks.get_networks(config, project) + timestamp = time.time() + for project_id in config["monitored_projects"]: + network_dict = networks.get_networks(config, project_id) current_quota_limit = limits.get_quota_current_limit( - config, f"projects/{project}", config["limit_names"]["GCE_INSTANCES"]) + config, f"projects/{project_id}", + config["limit_names"]["GCE_INSTANCES"]) if current_quota_limit is None: print( - f"Could not write number of instances for projects/{project} due to missing quotas" + f"Could not determine number of instances for projects/{project_id} due to missing quotas" ) current_quota_limit_view = metrics.customize_quota_view(current_quota_limit) @@ -81,15 +84,19 @@ def get_gce_instances_data(config, metrics_dict, gce_instance_dict, limit_dict): if net['self_link'] in gce_instance_dict: usage = gce_instance_dict[net['self_link']] - metrics.write_data_to_metric( - config, project, usage, metrics_dict["metrics_per_network"] - ["instance_per_network"]["usage"]["name"], net['network_name']) - metrics.write_data_to_metric( - config, project, net['limit'], metrics_dict["metrics_per_network"] - ["instance_per_network"]["limit"]["name"], net['network_name']) - metrics.write_data_to_metric( - config, project, usage / net['limit'], - metrics_dict["metrics_per_network"]["instance_per_network"] - ["utilization"]["name"], net['network_name']) + metric_labels = { + 'project': project_id, + 'network_name': net['network_name'] + } + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["instance_per_network"] + ["usage"]["name"], usage, metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["instance_per_network"] + ["limit"]["name"], net['limit'], metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["instance_per_network"] + ["utilization"]["name"], usage / net['limit'], metric_labels, + timestamp=timestamp) - print(f"Wrote number of instances to metric for projects/{project}") + print(f"Buffered number of instances to metric for projects/{project_id}") diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index a379a5c0..5c5bfc73 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -14,6 +14,8 @@ # limitations under the License. # +import time + from google.api_core import exceptions from google.cloud import monitoring_v3 from . import metrics @@ -173,6 +175,8 @@ def count_effective_limit(config, project_id, network_dict, usage_metric_name, None ''' + timestamp = time.time() + if network_dict['peerings'] == []: return @@ -215,11 +219,16 @@ def count_effective_limit(config, project_id, network_dict, usage_metric_name, # Calculates effective limit: Step 4: Find maximum from step 1 and step 3 effective_limit = max(limit_step1, limit_step3) utilization = peering_group_usage / effective_limit - - metrics.write_data_to_metric(config, project_id, peering_group_usage, - usage_metric_name, network_dict['network_name']) - metrics.write_data_to_metric(config, project_id, effective_limit, - limit_metric_name, network_dict['network_name']) - metrics.write_data_to_metric(config, project_id, utilization, - utilization_metric_name, - network_dict['network_name']) \ No newline at end of file + metric_labels = { + 'project': project_id, + 'network_name': network_dict['network_name'] + } + metrics.append_data_to_series_buffer(config, usage_metric_name, + peering_group_usage, metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer(config, limit_metric_name, + effective_limit, metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer(config, utilization_metric_name, + utilization, metric_labels, + timestamp=timestamp) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 155365b6..4792c27f 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -14,12 +14,16 @@ # limitations under the License. # +from curses import KEY_MARK +import re import time import yaml from google.api import metric_pb2 as ga_metric from google.cloud import monitoring_v3 from . import peerings, limits, networks +BUFFER_LEN = 10 + def create_metrics(monitoring_project): ''' @@ -84,35 +88,32 @@ def create_metric(metric_name, description, monitoring_project): print("Created {}.".format(descriptor.name)) -def write_data_to_metric(config, monitored_project_id, value, metric_name, - network_name=None, subnet_id=None): +def append_data_to_series_buffer(config, metric_name, metric_value, + metric_labels, timestamp=None): ''' Writes data to Cloud Monitoring custom metrics. Parameters: config (dict): The dict containing config like clients and limits - monitored_project_id: ID of the project where the resource lives (will be added as a label) - value (int): Value for the data point of the metric. metric_name (string): Name of the metric - network_name (string): Name of the network (will be added as a label) - subnet_id (string): Identifier of the Subnet (region/name of the subnet) + metric_value (int): Value for the data point of the metric. + matric_labels (dictionary of dictionary of string: string): metric labels names and values + timestamp (float): seconds since the epoch, in UTC Returns: usage (int): Current usage for that network. limit (int): Current usage for that network. ''' - client = monitoring_v3.MetricServiceClient() series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" - series.metric.labels["project"] = monitored_project_id - if network_name != None: - series.metric.labels["network_name"] = network_name - if subnet_id != None: - series.metric.labels["subnet_id"] = subnet_id - now = time.time() - seconds = int(now) - nanos = int((now - seconds) * 10**9) + for label_name in metric_labels: + if (metric_labels[label_name] != None): + series.metric.labels[label_name] = metric_labels[label_name] + + timestamp = timestamp if timestamp != None else time.time() + seconds = int(timestamp) + nanos = int((timestamp - seconds) * 10**9) interval = monitoring_v3.TimeInterval( {"end_time": { "seconds": seconds, @@ -121,20 +122,39 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name, point = monitoring_v3.Point({ "interval": interval, "value": { - "double_value": value + "double_value": metric_value } }) series.points = [point] # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error # Implement exponential backoff retries? + config["series_buffer"].append(series) + if len(config["series_buffer"]) >= BUFFER_LEN: + flush_series_buffer(config) + + +def flush_series_buffer(config): + ''' + writes buffered metrics to Google Cloud Monitoring, empties buffer upon failure + config (dict): The dict containing config like clients and limits + ''' try: - client.create_time_series(name=config["monitoring_project_link"], - time_series=[series]) + if config["series_buffer"] and len(config["series_buffer"]) > 0: + client = monitoring_v3.MetricServiceClient() + client.create_time_series(name=config["monitoring_project_link"], + time_series=config["series_buffer"]) + series_names = [ + re.search("\/(.+$)", series.metric.type).group(1) + for series in config["series_buffer"] + ] + print("Wrote time series: ", series_names) except Exception as e: - print("Error while writing data point for metric", metric_name) + print("Error while flushing series buffer") print(e) + config["series_buffer"] = [] + def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): ''' @@ -148,18 +168,18 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): Returns: None ''' - for project in config["monitored_projects"]: - network_dict_list = peerings.gather_peering_data(config, project) + for project_id in config["monitored_projects"]: + network_dict_list = peerings.gather_peering_data(config, project_id) # Network dict list is a list of dictionary (one for each network) # For each network, this dictionary contains: # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) # peerings is a list of dictionary (one for each peered network) and contains: # project_id, network_name, network_id current_quota_limit = limits.get_quota_current_limit( - config, f"projects/{project}", limit_metric) + config, f"projects/{project_id}", limit_metric) if current_quota_limit is None: print( - f"Could not write number of L7 forwarding rules to metric for projects/{project} due to missing quotas" + f"Could not determine number of L7 forwarding rules to metric for projects/{project_id} due to missing quotas" ) continue @@ -169,10 +189,10 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): for network_dict in network_dict_list: if network_dict['network_id'] == 0: print( - f"Could not write {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project} due to missing permissions." + f"Could not determine {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id} due to missing permissions." ) continue - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}" + network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" limit = networks.get_limit_network(network_dict, network_link, current_quota_limit_view, limit_dict) @@ -197,7 +217,7 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): limit_metric) if current_peered_quota_limit is None: print( - f"Could not write metrics for peering to projects/{peered_network_dict['project_id']} due to missing quotas" + f"Could not determine metrics for peering to projects/{peered_network_dict['project_id']} due to missing quotas" ) continue @@ -211,13 +231,13 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): peered_network_dict["usage"] = peered_usage peered_network_dict["limit"] = peered_limit - limits.count_effective_limit(config, project, network_dict, + limits.count_effective_limit(config, project_id, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_dict) print( - f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}" + f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" ) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py index 141364d1..616c7f66 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py @@ -14,6 +14,8 @@ # limitations under the License. # +import time + from . import metrics, networks, limits @@ -28,40 +30,53 @@ def get_vpc_peering_data(config, metrics_dict, limit_dict): Returns: None ''' + timestamp = time.time() for project in config["monitored_projects"]: active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data( config, project, limit_dict) + for peering in active_vpc_peerings: - metrics.write_data_to_metric( - config, project, peering['active_peerings'], - metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] - ["usage"]["name"], peering['network_name']) - metrics.write_data_to_metric( - config, project, peering['network_limit'], - metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] - ["limit"]["name"], peering['network_name']) - metrics.write_data_to_metric( - config, project, - peering['active_peerings'] / peering['network_limit'], - metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] - ["utilization"]["name"], peering['network_name']) - print("Wrote number of active VPC peerings to custom metric for project:", - project) + metric_labels = { + 'project': project, + 'network_name': peering['network_name'] + } + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["vpc_peering_active_per_network"]["usage"]["name"], + peering['active_peerings'], metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["vpc_peering_active_per_network"]["limit"]["name"], + peering['network_limit'], metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["vpc_peering_active_per_network"]["utilization"]["name"], + peering['active_peerings'] / peering['network_limit'], metric_labels, + timestamp=timestamp) + print( + "Buffered number of active VPC peerings to custom metric for project:", + project) for peering in vpc_peerings: - metrics.write_data_to_metric( - config, project, peering['peerings'], - metrics_dict["metrics_per_network"]["vpc_peering_per_network"] - ["usage"]["name"], peering['network_name']) - metrics.write_data_to_metric( - config, project, peering['network_limit'], - metrics_dict["metrics_per_network"]["vpc_peering_per_network"] - ["limit"]["name"], peering['network_name']) - metrics.write_data_to_metric( - config, project, peering['peerings'] / peering['network_limit'], - metrics_dict["metrics_per_network"]["vpc_peering_per_network"] - ["utilization"]["name"], peering['network_name']) - print("Wrote number of VPC peerings to custom metric for project:", project) + metric_labels = { + 'project': project, + 'network_name': peering['network_name'] + } + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["vpc_peering_per_network"] + ["usage"]["name"], peering['peerings'], metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["vpc_peering_per_network"] + ["limit"]["name"], peering['network_limit'], metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"]["vpc_peering_per_network"] + ["utilization"]["name"], + peering['peerings'] / peering['network_limit'], metric_labels, + timestamp=timestamp) + print("Buffered number of VPC peerings to custom metric for project:", + project) def gather_peering_data(config, project_id): diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py index fbe9ff3b..8bca3496 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py @@ -14,6 +14,8 @@ # limitations under the License. # +import time + from collections import defaultdict from . import metrics, networks, limits, peerings, routers @@ -88,16 +90,17 @@ def get_dynamic_routes(config, metrics_dict, limits_dict): routers_dict = routers.get_routers(config) dynamic_routes_dict = defaultdict(int) - for project_id in config["monitored_projects"]: - network_dict = networks.get_networks(config, project_id) + timestamp = time.time() + for project in config["monitored_projects"]: + network_dict = networks.get_networks(config, project) - for network in network_dict: - sum_routes = get_routes_for_network(config, network['self_link'], - project_id, routers_dict) - dynamic_routes_dict[network['self_link']] = sum_routes + for net in network_dict: + sum_routes = get_routes_for_network(config, net['self_link'], project, + routers_dict) + dynamic_routes_dict[net['self_link']] = sum_routes - if network['self_link'] in limits_dict: - limit = limits_dict[network['self_link']] + if net['self_link'] in limits_dict: + limit = limits_dict[net['self_link']] else: if 'default_value' in limits_dict: limit = limits_dict['default_value'] @@ -106,21 +109,21 @@ def get_dynamic_routes(config, metrics_dict, limits_dict): break utilization = sum_routes / limit + metric_labels = {'project': project, 'network_name': net['network_name']} + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["dynamic_routes_per_network"]["usage"]["name"], sum_routes, + metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["dynamic_routes_per_network"]["limit"]["name"], limit, metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_network"] + ["dynamic_routes_per_network"]["utilization"]["name"], utilization, + metric_labels, timestamp=timestamp) - metrics.write_data_to_metric( - config, project_id, sum_routes, metrics_dict["metrics_per_network"] - ["dynamic_routes_per_network"]["usage"]["name"], - network['network_name']) - metrics.write_data_to_metric( - config, project_id, limit, metrics_dict["metrics_per_network"] - ["dynamic_routes_per_network"]["limit"]["name"], - network['network_name']) - metrics.write_data_to_metric( - config, project_id, utilization, metrics_dict["metrics_per_network"] - ["dynamic_routes_per_network"]["utilization"]["name"], - network['network_name']) - - print("Wrote metrics for dynamic routes for VPCs in project", project_id) + print("Buffered metrics for dynamic routes for VPCs in project", project) return dynamic_routes_dict diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py index 8b173dc8..fc5c0490 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py @@ -14,6 +14,8 @@ # limitations under the License. # +import time + from . import metrics from google.protobuf import field_mask_pb2 from google.protobuf.json_format import MessageToDict @@ -225,6 +227,7 @@ def get_subnets(config, metrics_dict): # Updates all_subnets_dict with the IP utilization info compute_subnet_utilization(config, all_subnets_dict) + timestamp = time.time() for project_id in config["monitored_projects"]: if project_id not in all_subnets_dict: continue @@ -236,18 +239,23 @@ def get_subnets(config, metrics_dict): # Building unique identifier with subnet region/name subnet_id = f"{subnet_dict['region']}/{subnet_dict['name']}" - metrics.write_data_to_metric( - config, project_id, subnet_dict['used_ip_addresses'], - metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"]["usage"] - ["name"], subnet_dict['network_name'], subnet_id) - metrics.write_data_to_metric( - config, project_id, subnet_dict['total_ip_addresses'], - metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"]["limit"] - ["name"], subnet_dict['network_name'], subnet_id) - metrics.write_data_to_metric( - config, project_id, ip_utilization, metrics_dict["metrics_per_subnet"] - ["ip_usage_per_subnet"]["utilization"]["name"], - subnet_dict['network_name'], subnet_id) + metric_labels = { + 'project': project_id, + 'network_name': subnet_dict['network_name'], + 'subnet_id': subnet_id + } + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"] + ["usage"]["name"], subnet_dict['used_ip_addresses'], metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"] + ["limit"]["name"], subnet_dict['total_ip_addresses'], metric_labels, + timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"] + ["utilization"]["name"], ip_utilization, metric_labels, + timestamp=timestamp) - print("Wrote metrics for subnet ip utilization for VPCs in project", + print("Buffered metrics for subnet ip utilization for VPCs in project", project_id) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py index 8d15eacd..e7abade5 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -15,11 +15,13 @@ # import re +import time + from collections import defaultdict from pydoc import doc from collections import defaultdict from google.protobuf import field_mask_pb2 -from . import metrics, networks, limits, peerings, routers +from . import metrics, networks, limits def get_firewalls_dict(config: dict): @@ -69,43 +71,51 @@ def get_firewalls_data(config, metrics_dict, project_quotas_dict, Parameters: config (dict): The dict containing config like clients and limits metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. - firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. + firewalls_dict (dictionary of of dictionary of string: string): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules Returns: None ''' - for project in config["monitored_projects"]: - current_quota_limit = project_quotas_dict[project]['global']["firewalls"] + timestamp = time.time() + for project_id in config["monitored_projects"]: + + current_quota_limit = project_quotas_dict[project_id]['global']["firewalls"] if current_quota_limit is None: print( - f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas" + f"Could not determine VPC firewal rules to metric for projects/{project_id} due to missing quotas" ) continue - network_dict = networks.get_networks(config, project) + network_dict = networks.get_networks(config, project_id) project_usage = 0 for net in network_dict: usage = 0 - if project in firewalls_dict and net['network_name'] in firewalls_dict[ - project]: - usage = firewalls_dict[project][net['network_name']] + if project_id in firewalls_dict and net['network_name'] in firewalls_dict[ + project_id]: + usage = firewalls_dict[project_id][net['network_name']] project_usage += usage - metrics.write_data_to_metric( - config, project, usage, + metric_labels = { + 'project': project_id, + 'network_name': net['network_name'] + } + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_project"][f"firewalls"]["usage"]["name"], - net['network_name']) + usage, metric_labels, timestamp=timestamp) + metric_labels = {'project': project_id} # firewall quotas are per project, not per single VPC - metrics.write_data_to_metric( - config, project, current_quota_limit['limit'], - metrics_dict["metrics_per_project"][f"firewalls"]["limit"]["name"]) - metrics.write_data_to_metric( - config, project, project_usage / current_quota_limit['limit'] - if current_quota_limit['limit'] != 0 else 0, - metrics_dict["metrics_per_project"][f"firewalls"]["utilization"] - ["name"]) - + metrics.append_data_to_series_buffer( + config, + metrics_dict["metrics_per_project"][f"firewalls"]["limit"]["name"], + current_quota_limit['limit'], metric_labels, timestamp=timestamp) + metrics.append_data_to_series_buffer( + config, metrics_dict["metrics_per_project"][f"firewalls"]["utilization"] + ["name"], project_usage / current_quota_limit['limit'] + if current_quota_limit['limit'] != 0 else 0, metric_labels, + timestamp=timestamp) print( - f"Wrote number of VPC Firewall Rules to metric for projects/{project}") + f"Buffered number of VPC Firewall Rules to metric for projects/{project_id}" + ) From 3608b573fd371c0d0a29d0d1306624245c3c06ac Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Mon, 10 Oct 2022 17:45:08 +0200 Subject: [PATCH 10/14] updated dashbaord and readme --- .../network-dashboard/README.md | 3 +- .../dashboards/quotas-utilization.json | 99 ++++++++++++++++--- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 254c8055..5d5b5993 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -26,7 +26,7 @@ Clone this repository, then go through the following steps to create resources: Once the resources are deployed, go to the following page to see the dashboard: https://console.cloud.google.com/monitoring/dashboards?project=. A dashboard called "quotas-utilization" should be created. -The Cloud Function runs every 5 minutes by default so you should start getting some data points after a few minutes. +The Cloud Function runs every 5m minutes by default so you should start getting some data points after a few minutes. You can change this frequency by modifying the "schedule_cron" variable in variables.tf. Once done testing, you can clean up resources by running `terraform destroy`. @@ -46,6 +46,7 @@ The Cloud Function currently tracks usage, limit and utilization of: - Dynamic routes per VPC peering group - IP utilization per subnet (% of IP addresses used in a subnet) - VPC firewall rules per project (VPC drill down is available for usage) +- Tuples per Firewall Policy It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 923dbc84..e6aeba30 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -420,16 +420,11 @@ ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_vpc_usage\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_vpc_usage\" resource.type=\"global\"" } } } ], - "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -465,16 +460,11 @@ ], "perSeriesAligner": "ALIGN_MAX" }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } + "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"" } } } ], - "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -520,8 +510,91 @@ }, "width": 6, "xPos": 0, + "yPos": 28 + }, + { + "height": 4, + "widget": { + "title": "tuples_per_firewall_policy", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_usage\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } + } + } + } + ], + "thresholds": [], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 0, + "yPos": 24 + }, + { + "height": 4, + "widget": { + "title": "tuples_per_firewall_policy_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_NONE" + } + } + } + } + ], + "thresholds": [], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, "yPos": 24 } ] - } + }, + "name": "projects/361499891700/dashboards/41fd2fe5-243c-4f3a-8719-2e36e99f53c2" } \ No newline at end of file From 801fe1c626057c084fdfed3bb702c88a26b2583f Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 14 Oct 2022 12:55:07 +0200 Subject: [PATCH 11/14] fixes to dashboard --- .../dashboards/quotas-utilization.json | 279 +++++------------- 1 file changed, 82 insertions(+), 197 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 8081e6ae..0e71bb7f 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -1,7 +1,6 @@ { "category": "CUSTOM", - "category": "CUSTOM", - "displayName": "quotas_utilization", + "displayName": "quotas_utilization_updated", "mosaicLayout": { "columns": 12, "tiles": [ @@ -23,11 +22,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "1800s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -63,11 +64,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -103,11 +106,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -143,11 +148,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -183,11 +190,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_INTERPOLATE" } } @@ -223,11 +232,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -263,11 +274,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -303,11 +316,13 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" } } @@ -343,6 +358,7 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"" @@ -375,11 +391,11 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"" @@ -395,8 +411,8 @@ } }, "width": 6, - "xPos": 6, - "yPos": 16 + "xPos": 0, + "yPos": 20 }, { "height": 4, @@ -436,7 +452,7 @@ }, "width": 6, "xPos": 0, - "yPos": 20 + "yPos": 24 }, { "height": 4, @@ -476,12 +492,12 @@ }, "width": 6, "xPos": 6, - "yPos": 20 + "yPos": 24 }, { "height": 4, "widget": { - "title": "firewalls_per_project_vpc_usage", + "title": "tuples_per_firewall_policy_utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -496,13 +512,10 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [ - "metric.label.\"project\"" - ], + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_utilization\" resource.type=\"global\"" } } } @@ -521,7 +534,7 @@ { "height": 4, "widget": { - "title": "tuples_per_firewall_policy", + "title": "ip_addresses_per_subnet_utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -536,190 +549,62 @@ "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_usage\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } - }, - "width": 6, - "xPos": 0, - "yPos": 20 - }, - { - "height": 4, - "widget": { - "title": "firewalls_per_project_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MAX", - "groupByFields": [ - "metric.label.\"project\"" - ], - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"" - } - } - } - ], - "thresholds": [], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 6, - "yPos": 16 - }, - { - "height": 4, - "widget": { - "title": "firewalls_per_project_vpc_usage", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": [ - "metric.label.\"project\"" - ], - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 0, - "yPos": 28 - }, - { - "height": 4, - "widget": { - "title": "tuples_per_firewall_policy", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_usage\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } - } - } - } - ], -<<<<<<< HEAD -======= - "thresholds": [], ->>>>>>> 68980127fdbfb079f4b1a041234d750bc7175c84 - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 0, - "yPos": 24 - }, - { - "height": 4, - "widget": { - "title": "tuples_per_firewall_policy_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MAX", - "groupByFields": [ - "metric.label.\"project\"" - ], - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" - } - } - } - } - ], - "thresholds": [], - "thresholds": [], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 6, - "yPos": 24 + "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" + } } - ] + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" } - } \ No newline at end of file + } + }, + "width": 6, + "xPos": 6, + "yPos": 16 + }, + { + "height": 4, + "widget": { + "title": "dynamic_routes_ppg_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_peering_group_utilization\" resource.type=\"global\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, + "yPos": 20 + } + ] + } +} \ No newline at end of file From 2248639e8b74baf583aae5aa609e28c355f37f3e Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 14 Oct 2022 13:02:05 +0200 Subject: [PATCH 12/14] fixed merge --- .../network-dashboard/cloud-function/main.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index 8bf59c22..84bf007c 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -218,19 +218,6 @@ def main(event, context): return 'Function execution completed' -<<<<<<< HEAD -if __name__ == "__main__": - main(None, None) -======= -if CFv2: - - @functions_framework.http - def main_http(request): - main(None, None) -else: - if __name__ == "__main__": - main(None, None) ->>>>>>> 68980127fdbfb079f4b1a041234d750bc7175c84 if CFv2: @@ -239,4 +226,4 @@ if CFv2: main(None, None) else: if __name__ == "__main__": - main(None, None) \ No newline at end of file + main(None, None) From 4d11251f176f00de2d5273a10776bbdff94750d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Wed, 19 Oct 2022 16:40:42 +0200 Subject: [PATCH 13/14] Update README.md --- blueprints/cloud-operations/network-dashboard/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 5d5b5993..ece25e85 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -26,9 +26,12 @@ Clone this repository, then go through the following steps to create resources: Once the resources are deployed, go to the following page to see the dashboard: https://console.cloud.google.com/monitoring/dashboards?project=. A dashboard called "quotas-utilization" should be created. -The Cloud Function runs every 5m minutes by default so you should start getting some data points after a few minutes. +The Cloud Function runs every 10 minutes by default so you should start getting some data points after a few minutes. +You can use the metric explorer to view the data points for the different custom metrics created: https://console.cloud.google.com/monitoring/metrics-explorer?project=. You can change this frequency by modifying the "schedule_cron" variable in variables.tf. +Note that some charts in the dashboard align values over 1h so you might need to wait 1h to see charts on the dashboard views. + Once done testing, you can clean up resources by running `terraform destroy`. ## Supported limits and quotas @@ -59,4 +62,4 @@ In a future release, we could support: - Static routes per VPC / per VPC peering group - Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) -If you are interested in this and/or would like to contribute, please contact legranda@google.com. \ No newline at end of file +If you are interested in this and/or would like to contribute, please contact legranda@google.com. From d2e38fb9d667f46fe68f0a0150b599781bd25c49 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Wed, 19 Oct 2022 17:15:54 +0200 Subject: [PATCH 14/14] removed dependency --- .../cloud-function/metrics/firewall_policies.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py index 8c9e33a2..ef4c2518 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py @@ -14,8 +14,6 @@ # limitations under the License. # -from ast import AnnAssign - import re import time