zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support for project level VPC firewall metrics

This commit is contained in:
Maurizio Noseda Pedraglio 2022-10-03 15:48:46 +02:00
parent 4f6cf40c98
commit cdf53552d7
5 changed files with 239 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
blueprints/cloud-operations/network-dashboard/cloud-function/main.py
View File

@ -20,7 +20,7 @@ import time
from google.cloud import monitoring_v3, asset_v1
from google.protobuf import field_mask_pb2
from googleapiclient import discovery
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, subnets
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, subnets, vpc_firewalls
def get_monitored_projects_list(config):
@ -33,7 +33,7 @@ def get_monitored_projects_list(config):
monitored_projects (List of strings): Full list of projects to be monitored
'''
monitored_projects = config["monitored_projects"]
monitored_folders = os.environ.get("MONITORED_FOLDERS_LIST").split(",")
monitored_folders = [] #os.environ.get("MONITORED_FOLDERS_LIST").split(",")
# Handling empty monitored folders list
if monitored_folders == ['']:
@ -89,15 +89,18 @@ def monitoring_interval():
config = {
# Organization ID containing the projects to be monitored
"organization":
os.environ.get("ORGANIZATION_ID"),
"organization": #os.environ.get("ORGANIZATION_ID"),
'34855741773',
# list of projects from which function will get quotas information
"monitored_projects":
os.environ.get("MONITORED_PROJECTS_LIST").split(","),
"monitoring_project_link":
os.environ.get('MONITORING_PROJECT_ID'),
"monitoring_project_link":
f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
"monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","),
[
"mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0",
"mnoseda-dev-net-spoke-0"
],
"monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'),
"monitoring-tlc",
"monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
f"projects/monitoring-tlc",
"monitoring_interval":
monitoring_interval(),
"limit_names": {
@ -143,6 +146,9 @@ def main(event, context):
metrics_dict, limits_dict = metrics.create_metrics(
config["monitoring_project_link"])
project_quotas_dict = limits.get_quota_project_limit(config)
firewalls_dict = vpc_firewalls.get_firewalls_dict(config)
# IP utilization subnet level metrics
subnets.get_subnets(config, metrics_dict)
@ -153,6 +159,10 @@ def main(event, context):
l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7")
subnet_range_dict = networks.get_subnet_ranges_dict(config)
# Per Project metrics
vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict,
firewalls_dict)
# Per Network metrics
instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict,
limits_dict['number_of_instances_limit'])
@ -197,4 +207,4 @@ def main(event, context):
if __name__ == "__main__":
main(None, None)
main(None, None)

14
blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml
View File

@ -159,4 +159,16 @@ metrics_per_peering_group:
default_value: 300
utilization:
name: dynamic_routes_per_peering_group_utilization
description: Number of Dynamic routes per peering group - utilization.
description: Number of Dynamic routes per peering group - utilization.
metrics_per_project:
firewalls:
usage:
name: firewalls_per_project_vpc_usage
description: Number of VPC firewall rules in a project - usage.
limit:
#no default limit as we can assume quotas can be always read from gcloud API
name: firewalls_per_project_limit
description: Number of VPC firewall rules in a project - limit.
utilization:
name: firewalls_per_project_utilization
description: Number of VPC firewall rules in a project - utilization.

54
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py
View File

@ -19,6 +19,60 @@ from google.cloud import monitoring_v3
from . import metrics
def get_quotas_dict(quotas_list):
'''
Creates a dictionary of quotas from a list, with lowe case keys
Parameters:
quotas_array (array): array of quotas
Returns:
quotas_dict (dict): dictionary of quotas
'''
quota_keys = [q['metric'] for q in quotas_list]
quotas_dict = dict()
i = 0
for key in quota_keys:
if ("metric" in quotas_list[i]):
del (quotas_list[i]["metric"])
quotas_dict[key.lower()] = quotas_list[i]
i += 1
return quotas_dict
def get_quota_project_limit(config, regions=["global"]):
'''
Retrieves limit for a specific project quota
Parameters:
project_link (string): Project link.
Returns:
quotas (dict): quotas for all selected regions, default 'global'
'''
try:
request = {}
quotas = dict()
for project in config["monitored_projects"]:
quotas[project] = dict()
if regions != ["global"]:
for region in regions:
request = config["clients"]["discovery_client"].compute.regions().get(
region=region, project=project)
response = request.execute()
quotas[project][region] = get_quotas_dict(response['quotas'])
else:
region = "global"
request = config["clients"]["discovery_client"].projects().get(
project=project, fields="quotas")
response = request.execute()
quotas[project][region] = get_quotas_dict(response['quotas'])
return quotas
except exceptions.PermissionDenied as err:
print(
f"Warning: error reading quotas for {project}. " +
f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
)
return None
def get_ppg(network_link, limit_dict):
'''
Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.

14
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py
View File

@ -36,7 +36,9 @@ def create_metrics(monitoring_project):
existing_metrics.append(desc.type)
limits_dict = {}
with open("metrics.yaml", 'r') as stream:
with open(
"/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml",
'r') as stream:
try:
metrics_dict = yaml.safe_load(stream)
@ -52,8 +54,9 @@ def create_metrics(monitoring_project):
# Subnet level metrics have a different limit: the subnet IP range size
if sub_metric_key == "limit" and metric_name != "ip_usage_per_subnet":
limits_dict_for_metric = {}
for network_link, limit_value in sub_metric["values"].items():
limits_dict_for_metric[network_link] = limit_value
if "values" in sub_metric:
for network_link, limit_value in sub_metric["values"].items():
limits_dict_for_metric[network_link] = limit_value
limits_dict[sub_metric["name"]] = limits_dict_for_metric
return metrics_dict, limits_dict
@ -84,7 +87,7 @@ def create_metric(metric_name, description, monitoring_project):
def write_data_to_metric(config, monitored_project_id, value, metric_name,
network_name, subnet_id=None):
network_name=None, subnet_id=None):
'''
Writes data to Cloud Monitoring custom metrics.
Parameters:
@ -103,7 +106,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name,
series = monitoring_v3.TimeSeries()
series.metric.type = f"custom.googleapis.com/{metric_name}"
series.resource.type = "global"
series.metric.labels["network_name"] = network_name
if network_name != None:
series.metric.labels["network_name"] = network_name
series.metric.labels["project"] = monitored_project_id
if subnet_id:
series.metric.labels["subnet_id"] = subnet_id

142
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py Normal file
View File

@ -0,0 +1,142 @@
#
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
from collections import defaultdict
from pydoc import doc
from collections import defaultdict
from google.protobuf import field_mask_pb2
from . import metrics, networks, limits, peerings, routers
def get_firewalls_dict(config: dict):
'''
Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization.
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules
'''
firewalls_dict = defaultdict(int)
read_mask = field_mask_pb2.FieldMask()
read_mask.FromJsonString('name,versionedResources')
response = config["clients"]["asset_client"].search_all_resources(
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ["compute.googleapis.com/Firewall"],
"read_mask": read_mask,
})
for resource in response:
project_id = re.search("(compute.googleapis.com/projects/)([\w\-\d]+)",
resource.name).group(2)
network_name = ""
for versioned in resource.versioned_resources:
for field_name, field_value in versioned.resource.items():
if field_name == "network":
network_name = re.search("[a-z0-9\-]*$", field_value).group(0)
firewalls_dict[project_id] = defaultdict(
int
) if not project_id in firewalls_dict else firewalls_dict[project_id]
firewalls_dict[project_id][
network_name] = 1 if not network_name in firewalls_dict[
project_id] else firewalls_dict[project_id][network_name] + 1
break
break
return firewalls_dict
'''def get_firewalls_per_project_vpc(config, project_id ):
Returns returns the VPC firewall rules defined in a project
Parameters:
config (dict): The dict containing config like clients and limits
project_id (string): Project ID for the project containing the Cloud Router.
Returns:
sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project
get_firewalls_dict(config)
sum_firewalls_per_vpc=dict()
sum_firewalls=0
page_token=None
while (1):
request = config["clients"]["discovery_client"].firewalls().list(
project=project_id,pageToken=page_token)
response = request.execute()
sum_firewalls_per_vpc = dict()
if 'items' in response:
for firewall_rule in response['items']:
sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1
else:
break
page_token = response['pageToken'] if 'pageToken' in response else None
if (not page_token):
break
return sum_firewalls_per_vpc
'''
def get_firewalls_data(config, metrics_dict, project_quotas_dict,
firewalls_dict):
'''
Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric.
Parameters:
config (dict): The dict containing config like clients and limits
metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules
Returns:
None
'''
for project in config["monitored_projects"]:
current_quota_limit = project_quotas_dict[project]['global']["firewalls"]
if current_quota_limit is None:
print(
f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas"
)
continue
network_dict = networks.get_networks(config, project)
project_usage = 0
for net in network_dict:
usage = 0
if net['network_name'] in firewalls_dict:
usage = firewalls_dict[project][net['network_name']]
project_usage += usage
metrics.write_data_to_metric(
config, project, usage,
metrics_dict["metrics_per_project"][f"firewalls"]["usage"]["name"],
net['network_name'])
#firewall quotas are per project, not per single VPC
metrics.write_data_to_metric(
config, project, current_quota_limit['limit'],
metrics_dict["metrics_per_project"][f"firewalls"]["limit"]["name"])
metrics.write_data_to_metric(
config, project, project_usage / current_quota_limit['limit']
if current_quota_limit['limit'] != 0 else 0,
metrics_dict["metrics_per_project"][f"firewalls"]["utilization"]
["name"])
print(
f"Wrote number of VPC Firewall Rules to metric for projects/{project}")