zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tracking of vpc firewalls, initial commit

This commit is contained in:
Maurizio Noseda Pedraglio 2022-09-30 11:10:48 +02:00
parent eea4642cf2
commit 1c134a735b
5 changed files with 234 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
blueprints/cloud-operations/network-dashboard/cloud-function/main.py
View File

@ -22,7 +22,7 @@ import time
from google.cloud import monitoring_v3, asset_v1
from google.protobuf import field_mask_pb2
from googleapiclient import discovery
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, vpc_firewalls
def monitoring_interval():
@ -49,15 +49,18 @@ def monitoring_interval():
config = {
# Organization ID containing the projects to be monitored
"organization":
os.environ.get("ORGANIZATION_ID"),
"organization": #os.environ.get("ORGANIZATION_ID"),
'34855741773',
# list of projects from which function will get quotas information
"monitored_projects":
os.environ.get("MONITORED_PROJECTS_LIST").split(","),
"monitoring_project_link":
os.environ.get('MONITORING_PROJECT_ID'),
"monitoring_project_link":
f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
"monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","),
[
"mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0",
"mnoseda-dev-net-spoke-0"
],
"monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'),
"monitoring-tlc",
"monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
f"projects/monitoring-tlc",
"monitoring_interval":
monitoring_interval(),
"limit_names": {
@ -98,6 +101,9 @@ def main(event, context):
metrics_dict, limits_dict = metrics.create_metrics(
config["monitoring_project_link"])
project_quotas_dict = limits.get_quota_project_limit(config)
firewalls_dict = vpc_firewalls.get_firewalls_dict(config)
# Asset inventory queries
gce_instance_dict = instances.get_gce_instance_dict(config)
@ -105,6 +111,10 @@ def main(event, context):
l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7")
subnet_range_dict = networks.get_subnet_ranges_dict(config)
# Per Project metrics
vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict,
firewalls_dict)
# Per Network metrics
instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict,
limits_dict['number_of_instances_limit'])

12
blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml
View File

@ -149,3 +149,15 @@ metrics_per_peering_group:
utilization:
name: dynamic_routes_per_peering_group_utilization
description: Number of Dynamic routes per peering group - utilization.
metrics_per_project:
firewalls:
usage:
name: firewalls_per_project_vpc_usage
description: Number of VPC firewall rules in a project - usage.
limit:
#no default limit as we can assume quotas can be always read from gcloud API
name: firewalls_per_project_limit
description: Number of VPC firewall rules in a project - limit.
utilization:
name: firewalls_per_project_utilization
description: Number of VPC firewall rules in a project - utilization.

55
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py
View File

@ -14,6 +14,8 @@
# limitations under the License.
#
from http.cookiejar import LWPCookieJar
from urllib import response
from google.api_core import exceptions
from google.cloud import monitoring_v3
from . import metrics
@ -71,6 +73,59 @@ def set_limits(network_dict, quota_limit, limit_dict):
network_dict['limit'] = 0
def get_quotas_dict(quotas_list):
'''
Creates a dictionary of quotas from a list, with lowe case keys
Parameters:
quotas_array (array): array of quotas
Returns:
quotas_dict (dict): dictionary of quotas
'''
quota_keys=[q['metric'] for q in quotas_list]
quotas_dict=dict()
i=0
for key in quota_keys:
if ("metric" in quotas_list[i] ):
del(quotas_list[i]["metric"])
quotas_dict[key.lower()]=quotas_list[i]
i+=1
return quotas_dict
def get_quota_project_limit(config,regions=["global"]):
'''
Retrieves limit for a specific project quota
Parameters:
project_link (string): Project link.
Returns:
quotas (dict): quotas for all selected regions, default 'global'
'''
try:
request={}
quotas=dict()
for project in config["monitored_projects"]:
quotas[project]=dict()
if regions != ["global"]:
for region in regions:
request=config["clients"]["discovery_client"].compute.regions().get(region=region,project=project)
response=request.execute()
quotas[project][region]=get_quotas_dict(response['quotas'])
else:
region="global"
request=config["clients"]["discovery_client"].projects().get(project=project,fields="quotas")
response=request.execute()
quotas[project][region]=get_quotas_dict(response['quotas'])
return quotas
except exceptions.PermissionDenied as err:
print(
f"Warning: error reading quotas for {project}. " +
f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
)
return None
#def get_project_quota_current_limit(config,project_link,metric_name,)
def get_quota_current_limit(config, project_link, metric_name):
'''
Retrieves limit for a specific metric.

12
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py
View File

@ -37,7 +37,7 @@ def create_metrics(monitoring_project):
existing_metrics.append(desc.type)
limits_dict = {}
with open("metrics.yaml", 'r') as stream:
with open("/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", 'r') as stream: #f
try:
metrics_dict = yaml.safe_load(stream)
@ -52,8 +52,9 @@ def create_metrics(monitoring_project):
# Parse limits (both default values and network specific ones)
if sub_metric_key == "limit":
limits_dict_for_metric = {}
for network_link, limit_value in sub_metric["values"].items():
limits_dict_for_metric[network_link] = limit_value
if "values" in sub_metric:
for network_link, limit_value in sub_metric["values"].items():
limits_dict_for_metric[network_link] = limit_value
limits_dict[sub_metric["name"]] = limits_dict_for_metric
return metrics_dict, limits_dict
@ -85,7 +86,7 @@ def create_metric(metric_name, description, monitoring_project):
def write_data_to_metric(config, monitored_project_id, value, metric_name,
network_name):
network_name=None):
'''
Writes data to Cloud Monitoring custom metrics.
@ -104,7 +105,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name,
series = monitoring_v3.TimeSeries()
series.metric.type = f"custom.googleapis.com/{metric_name}"
series.resource.type = "global"
series.metric.labels["network_name"] = network_name
if network_name:
series.metric.labels["network_name"] = network_name
series.metric.labels["project"] = monitored_project_id
now = time.time()

141
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py Normal file
View File

@ -0,0 +1,141 @@
#
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
from collections import defaultdict
from pydoc import doc
from collections import defaultdict
from google.protobuf import field_mask_pb2
from . import metrics, networks, limits, peerings, routers
def get_firewalls_dict(config: dict):
'''
Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization.
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules
'''
firewalls_dict = defaultdict(int)
read_mask = field_mask_pb2.FieldMask()
read_mask.FromJsonString('name,versionedResources')
response = config["clients"]["asset_client"].search_all_resources(
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ["compute.googleapis.com/Firewall"],
"read_mask": read_mask,
})
for resource in response:
project_id=re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", resource.name).group(2)
network_name=""
for versioned in resource.versioned_resources:
for field_name, field_value in versioned.resource.items():
if field_name == "network":
network_name = re.search("[a-z0-9\-]*$", field_value).group(0)
firewalls_dict[project_id]= defaultdict(int) if not project_id in firewalls_dict else firewalls_dict[project_id]
firewalls_dict[project_id][network_name]=1 if not network_name in firewalls_dict[project_id] else firewalls_dict[project_id][network_name]+1
break
break
return firewalls_dict
'''def get_firewalls_per_project_vpc(config, project_id ):
Returns returns the VPC firewall rules defined in a project
Parameters:
config (dict): The dict containing config like clients and limits
project_id (string): Project ID for the project containing the Cloud Router.
Returns:
sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project
get_firewalls_dict(config)
sum_firewalls_per_vpc=dict()
sum_firewalls=0
page_token=None
while (1):
request = config["clients"]["discovery_client"].firewalls().list(
project=project_id,pageToken=page_token)
response = request.execute()
sum_firewalls_per_vpc = dict()
if 'items' in response:
for firewall_rule in response['items']:
sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1
else:
break
page_token = response['pageToken'] if 'pageToken' in response else None
if (not page_token):
break
return sum_firewalls_per_vpc
'''
def get_firewalls_data(config, metrics_dict,project_quotas_dict,firewalls_dict):
'''
Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric.
Parameters:
config (dict): The dict containing config like clients and limits
metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules
Returns:
None
'''
for project in config["monitored_projects"]:
current_quota_limit= project_quotas_dict[project]['global']["firewalls"]
if current_quota_limit is None:
print(
f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas"
)
continue
network_dict = networks.get_networks(config, project)
project_usage=0
for net in network_dict:
usage = 0
if net['network_name'] in firewalls_dict:
usage = firewalls_dict[project][net['network_name']]
project_usage+=usage
metrics.write_data_to_metric(
config, project, usage, metrics_dict["metrics_per_project"]
[f"firewalls"]["usage"]["name"],
net['network_name'])
#firewall quotas are per project, not per single VPC
metrics.write_data_to_metric(
config, project, current_quota_limit['limit'], metrics_dict["metrics_per_project"]
[f"firewalls"]["limit"]["name"],
)
metrics.write_data_to_metric(
config, project, project_usage / current_quota_limit['limit'] if current_quota_limit['limit']!=0 else 0,
metrics_dict["metrics_per_project"]
[f"firewalls"]["utilization"]["name"],)
print(
f"Wrote number of VPC Firewall Rules to metric for projects/{project}"
)