From 1c134a735be4c532f44cde595a21ef639530c804 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 30 Sep 2022 11:10:48 +0200 Subject: [PATCH] tracking of vpc firewalls, initial commit --- .../network-dashboard/cloud-function/main.py | 28 ++-- .../cloud-function/metrics.yaml | 12 ++ .../cloud-function/metrics/limits.py | 55 +++++++ .../cloud-function/metrics/metrics.py | 12 +- .../cloud-function/metrics/vpc_firewalls.py | 141 ++++++++++++++++++ 5 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index ecb618ad..749762a6 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -22,7 +22,7 @@ import time from google.cloud import monitoring_v3, asset_v1 from google.protobuf import field_mask_pb2 from googleapiclient import discovery -from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes +from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, vpc_firewalls def monitoring_interval(): @@ -49,15 +49,18 @@ def monitoring_interval(): config = { # Organization ID containing the projects to be monitored - "organization": - os.environ.get("ORGANIZATION_ID"), + "organization": #os.environ.get("ORGANIZATION_ID"), + '34855741773', # list of projects from which function will get quotas information - "monitored_projects": - os.environ.get("MONITORED_PROJECTS_LIST").split(","), - "monitoring_project_link": - os.environ.get('MONITORING_PROJECT_ID'), - "monitoring_project_link": - f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", + "monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","), + [ + "mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0", + "mnoseda-dev-net-spoke-0" + ], + "monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'), + "monitoring-tlc", + "monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", + f"projects/monitoring-tlc", "monitoring_interval": monitoring_interval(), "limit_names": { @@ -98,6 +101,9 @@ def main(event, context): metrics_dict, limits_dict = metrics.create_metrics( config["monitoring_project_link"]) + project_quotas_dict = limits.get_quota_project_limit(config) + + firewalls_dict = vpc_firewalls.get_firewalls_dict(config) # Asset inventory queries gce_instance_dict = instances.get_gce_instance_dict(config) @@ -105,6 +111,10 @@ def main(event, context): l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") subnet_range_dict = networks.get_subnet_ranges_dict(config) + # Per Project metrics + vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, + firewalls_dict) + # Per Network metrics instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, limits_dict['number_of_instances_limit']) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 2e5621d7..9a2d6044 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -149,3 +149,15 @@ metrics_per_peering_group: utilization: name: dynamic_routes_per_peering_group_utilization description: Number of Dynamic routes per peering group - utilization. +metrics_per_project: + firewalls: + usage: + name: firewalls_per_project_vpc_usage + description: Number of VPC firewall rules in a project - usage. + limit: + #no default limit as we can assume quotas can be always read from gcloud API + name: firewalls_per_project_limit + description: Number of VPC firewall rules in a project - limit. + utilization: + name: firewalls_per_project_utilization + description: Number of VPC firewall rules in a project - utilization. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index 8b3d5ae9..c73d15d3 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -14,6 +14,8 @@ # limitations under the License. # +from http.cookiejar import LWPCookieJar +from urllib import response from google.api_core import exceptions from google.cloud import monitoring_v3 from . import metrics @@ -71,6 +73,59 @@ def set_limits(network_dict, quota_limit, limit_dict): network_dict['limit'] = 0 +def get_quotas_dict(quotas_list): + ''' + Creates a dictionary of quotas from a list, with lowe case keys + Parameters: + quotas_array (array): array of quotas + Returns: + quotas_dict (dict): dictionary of quotas + ''' + quota_keys=[q['metric'] for q in quotas_list] + quotas_dict=dict() + i=0 + for key in quota_keys: + if ("metric" in quotas_list[i] ): + del(quotas_list[i]["metric"]) + quotas_dict[key.lower()]=quotas_list[i] + i+=1 + return quotas_dict + + + +def get_quota_project_limit(config,regions=["global"]): + ''' + Retrieves limit for a specific project quota + Parameters: + project_link (string): Project link. + Returns: + quotas (dict): quotas for all selected regions, default 'global' + ''' + try: + request={} + quotas=dict() + for project in config["monitored_projects"]: + quotas[project]=dict() + if regions != ["global"]: + for region in regions: + request=config["clients"]["discovery_client"].compute.regions().get(region=region,project=project) + response=request.execute() + quotas[project][region]=get_quotas_dict(response['quotas']) + else: + region="global" + request=config["clients"]["discovery_client"].projects().get(project=project,fields="quotas") + response=request.execute() + quotas[project][region]=get_quotas_dict(response['quotas']) + + return quotas + except exceptions.PermissionDenied as err: + print( + f"Warning: error reading quotas for {project}. " + + f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" + ) + return None + +#def get_project_quota_current_limit(config,project_link,metric_name,) def get_quota_current_limit(config, project_link, metric_name): ''' Retrieves limit for a specific metric. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 14183f15..4a6267ed 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -37,7 +37,7 @@ def create_metrics(monitoring_project): existing_metrics.append(desc.type) limits_dict = {} - with open("metrics.yaml", 'r') as stream: + with open("/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", 'r') as stream: #f try: metrics_dict = yaml.safe_load(stream) @@ -52,8 +52,9 @@ def create_metrics(monitoring_project): # Parse limits (both default values and network specific ones) if sub_metric_key == "limit": limits_dict_for_metric = {} - for network_link, limit_value in sub_metric["values"].items(): - limits_dict_for_metric[network_link] = limit_value + if "values" in sub_metric: + for network_link, limit_value in sub_metric["values"].items(): + limits_dict_for_metric[network_link] = limit_value limits_dict[sub_metric["name"]] = limits_dict_for_metric return metrics_dict, limits_dict @@ -85,7 +86,7 @@ def create_metric(metric_name, description, monitoring_project): def write_data_to_metric(config, monitored_project_id, value, metric_name, - network_name): + network_name=None): ''' Writes data to Cloud Monitoring custom metrics. @@ -104,7 +105,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name, series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" - series.metric.labels["network_name"] = network_name + if network_name: + series.metric.labels["network_name"] = network_name series.metric.labels["project"] = monitored_project_id now = time.time() diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py new file mode 100644 index 00000000..1bc53da6 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -0,0 +1,141 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from collections import defaultdict +from pydoc import doc +from collections import defaultdict +from google.protobuf import field_mask_pb2 +from . import metrics, networks, limits, peerings, routers + + + + +def get_firewalls_dict(config: dict): + ''' + Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization. + + Parameters: + config (dict): The dict containing config like clients and limits + Returns: + firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + ''' + + firewalls_dict = defaultdict(int) + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + + response = config["clients"]["asset_client"].search_all_resources( + request={ + "scope": f"organizations/{config['organization']}", + "asset_types": ["compute.googleapis.com/Firewall"], + "read_mask": read_mask, + }) + for resource in response: + project_id=re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", resource.name).group(2) + network_name="" + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "network": + network_name = re.search("[a-z0-9\-]*$", field_value).group(0) + firewalls_dict[project_id]= defaultdict(int) if not project_id in firewalls_dict else firewalls_dict[project_id] + firewalls_dict[project_id][network_name]=1 if not network_name in firewalls_dict[project_id] else firewalls_dict[project_id][network_name]+1 + break + break + return firewalls_dict + + + +'''def get_firewalls_per_project_vpc(config, project_id ): + + Returns returns the VPC firewall rules defined in a project + + Parameters: + config (dict): The dict containing config like clients and limits + project_id (string): Project ID for the project containing the Cloud Router. + Returns: + sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project + + get_firewalls_dict(config) + sum_firewalls_per_vpc=dict() + sum_firewalls=0 + page_token=None + while (1): + request = config["clients"]["discovery_client"].firewalls().list( + project=project_id,pageToken=page_token) + response = request.execute() + sum_firewalls_per_vpc = dict() + if 'items' in response: + for firewall_rule in response['items']: + sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1 + else: + break + page_token = response['pageToken'] if 'pageToken' in response else None + if (not page_token): + break + + return sum_firewalls_per_vpc +''' + +def get_firewalls_data(config, metrics_dict,project_quotas_dict,firewalls_dict): + ''' + Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric. + + Parameters: + config (dict): The dict containing config like clients and limits + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. + firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + Returns: + None + ''' + for project in config["monitored_projects"]: + + current_quota_limit= project_quotas_dict[project]['global']["firewalls"] + if current_quota_limit is None: + print( + f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas" + ) + continue + + + network_dict = networks.get_networks(config, project) + + project_usage=0 + for net in network_dict: + usage = 0 + if net['network_name'] in firewalls_dict: + usage = firewalls_dict[project][net['network_name']] + project_usage+=usage + metrics.write_data_to_metric( + config, project, usage, metrics_dict["metrics_per_project"] + [f"firewalls"]["usage"]["name"], + net['network_name']) + + #firewall quotas are per project, not per single VPC + metrics.write_data_to_metric( + config, project, current_quota_limit['limit'], metrics_dict["metrics_per_project"] + [f"firewalls"]["limit"]["name"], + ) + metrics.write_data_to_metric( + config, project, project_usage / current_quota_limit['limit'] if current_quota_limit['limit']!=0 else 0, + metrics_dict["metrics_per_project"] + [f"firewalls"]["utilization"]["name"],) + + print( + f"Wrote number of VPC Firewall Rules to metric for projects/{project}" + )