tracking of vpc firewalls, initial commit
This commit is contained in:
parent
eea4642cf2
commit
1c134a735b
|
@ -22,7 +22,7 @@ import time
|
|||
from google.cloud import monitoring_v3, asset_v1
|
||||
from google.protobuf import field_mask_pb2
|
||||
from googleapiclient import discovery
|
||||
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes
|
||||
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, vpc_firewalls
|
||||
|
||||
|
||||
def monitoring_interval():
|
||||
|
@ -49,15 +49,18 @@ def monitoring_interval():
|
|||
|
||||
config = {
|
||||
# Organization ID containing the projects to be monitored
|
||||
"organization":
|
||||
os.environ.get("ORGANIZATION_ID"),
|
||||
"organization": #os.environ.get("ORGANIZATION_ID"),
|
||||
'34855741773',
|
||||
# list of projects from which function will get quotas information
|
||||
"monitored_projects":
|
||||
os.environ.get("MONITORED_PROJECTS_LIST").split(","),
|
||||
"monitoring_project_link":
|
||||
os.environ.get('MONITORING_PROJECT_ID'),
|
||||
"monitoring_project_link":
|
||||
f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
|
||||
"monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","),
|
||||
[
|
||||
"mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0",
|
||||
"mnoseda-dev-net-spoke-0"
|
||||
],
|
||||
"monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'),
|
||||
"monitoring-tlc",
|
||||
"monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}",
|
||||
f"projects/monitoring-tlc",
|
||||
"monitoring_interval":
|
||||
monitoring_interval(),
|
||||
"limit_names": {
|
||||
|
@ -98,6 +101,9 @@ def main(event, context):
|
|||
|
||||
metrics_dict, limits_dict = metrics.create_metrics(
|
||||
config["monitoring_project_link"])
|
||||
project_quotas_dict = limits.get_quota_project_limit(config)
|
||||
|
||||
firewalls_dict = vpc_firewalls.get_firewalls_dict(config)
|
||||
|
||||
# Asset inventory queries
|
||||
gce_instance_dict = instances.get_gce_instance_dict(config)
|
||||
|
@ -105,6 +111,10 @@ def main(event, context):
|
|||
l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7")
|
||||
subnet_range_dict = networks.get_subnet_ranges_dict(config)
|
||||
|
||||
# Per Project metrics
|
||||
vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict,
|
||||
firewalls_dict)
|
||||
|
||||
# Per Network metrics
|
||||
instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict,
|
||||
limits_dict['number_of_instances_limit'])
|
||||
|
|
|
@ -149,3 +149,15 @@ metrics_per_peering_group:
|
|||
utilization:
|
||||
name: dynamic_routes_per_peering_group_utilization
|
||||
description: Number of Dynamic routes per peering group - utilization.
|
||||
metrics_per_project:
|
||||
firewalls:
|
||||
usage:
|
||||
name: firewalls_per_project_vpc_usage
|
||||
description: Number of VPC firewall rules in a project - usage.
|
||||
limit:
|
||||
#no default limit as we can assume quotas can be always read from gcloud API
|
||||
name: firewalls_per_project_limit
|
||||
description: Number of VPC firewall rules in a project - limit.
|
||||
utilization:
|
||||
name: firewalls_per_project_utilization
|
||||
description: Number of VPC firewall rules in a project - utilization.
|
||||
|
|
|
@ -14,6 +14,8 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
from http.cookiejar import LWPCookieJar
|
||||
from urllib import response
|
||||
from google.api_core import exceptions
|
||||
from google.cloud import monitoring_v3
|
||||
from . import metrics
|
||||
|
@ -71,6 +73,59 @@ def set_limits(network_dict, quota_limit, limit_dict):
|
|||
network_dict['limit'] = 0
|
||||
|
||||
|
||||
def get_quotas_dict(quotas_list):
|
||||
'''
|
||||
Creates a dictionary of quotas from a list, with lowe case keys
|
||||
Parameters:
|
||||
quotas_array (array): array of quotas
|
||||
Returns:
|
||||
quotas_dict (dict): dictionary of quotas
|
||||
'''
|
||||
quota_keys=[q['metric'] for q in quotas_list]
|
||||
quotas_dict=dict()
|
||||
i=0
|
||||
for key in quota_keys:
|
||||
if ("metric" in quotas_list[i] ):
|
||||
del(quotas_list[i]["metric"])
|
||||
quotas_dict[key.lower()]=quotas_list[i]
|
||||
i+=1
|
||||
return quotas_dict
|
||||
|
||||
|
||||
|
||||
def get_quota_project_limit(config,regions=["global"]):
|
||||
'''
|
||||
Retrieves limit for a specific project quota
|
||||
Parameters:
|
||||
project_link (string): Project link.
|
||||
Returns:
|
||||
quotas (dict): quotas for all selected regions, default 'global'
|
||||
'''
|
||||
try:
|
||||
request={}
|
||||
quotas=dict()
|
||||
for project in config["monitored_projects"]:
|
||||
quotas[project]=dict()
|
||||
if regions != ["global"]:
|
||||
for region in regions:
|
||||
request=config["clients"]["discovery_client"].compute.regions().get(region=region,project=project)
|
||||
response=request.execute()
|
||||
quotas[project][region]=get_quotas_dict(response['quotas'])
|
||||
else:
|
||||
region="global"
|
||||
request=config["clients"]["discovery_client"].projects().get(project=project,fields="quotas")
|
||||
response=request.execute()
|
||||
quotas[project][region]=get_quotas_dict(response['quotas'])
|
||||
|
||||
return quotas
|
||||
except exceptions.PermissionDenied as err:
|
||||
print(
|
||||
f"Warning: error reading quotas for {project}. " +
|
||||
f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
|
||||
)
|
||||
return None
|
||||
|
||||
#def get_project_quota_current_limit(config,project_link,metric_name,)
|
||||
def get_quota_current_limit(config, project_link, metric_name):
|
||||
'''
|
||||
Retrieves limit for a specific metric.
|
||||
|
|
|
@ -37,7 +37,7 @@ def create_metrics(monitoring_project):
|
|||
existing_metrics.append(desc.type)
|
||||
limits_dict = {}
|
||||
|
||||
with open("metrics.yaml", 'r') as stream:
|
||||
with open("/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", 'r') as stream: #f
|
||||
try:
|
||||
metrics_dict = yaml.safe_load(stream)
|
||||
|
||||
|
@ -52,8 +52,9 @@ def create_metrics(monitoring_project):
|
|||
# Parse limits (both default values and network specific ones)
|
||||
if sub_metric_key == "limit":
|
||||
limits_dict_for_metric = {}
|
||||
for network_link, limit_value in sub_metric["values"].items():
|
||||
limits_dict_for_metric[network_link] = limit_value
|
||||
if "values" in sub_metric:
|
||||
for network_link, limit_value in sub_metric["values"].items():
|
||||
limits_dict_for_metric[network_link] = limit_value
|
||||
limits_dict[sub_metric["name"]] = limits_dict_for_metric
|
||||
|
||||
return metrics_dict, limits_dict
|
||||
|
@ -85,7 +86,7 @@ def create_metric(metric_name, description, monitoring_project):
|
|||
|
||||
|
||||
def write_data_to_metric(config, monitored_project_id, value, metric_name,
|
||||
network_name):
|
||||
network_name=None):
|
||||
'''
|
||||
Writes data to Cloud Monitoring custom metrics.
|
||||
|
||||
|
@ -104,7 +105,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name,
|
|||
series = monitoring_v3.TimeSeries()
|
||||
series.metric.type = f"custom.googleapis.com/{metric_name}"
|
||||
series.resource.type = "global"
|
||||
series.metric.labels["network_name"] = network_name
|
||||
if network_name:
|
||||
series.metric.labels["network_name"] = network_name
|
||||
series.metric.labels["project"] = monitored_project_id
|
||||
|
||||
now = time.time()
|
||||
|
|
|
@ -0,0 +1,141 @@
|
|||
#
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pydoc import doc
|
||||
from collections import defaultdict
|
||||
from google.protobuf import field_mask_pb2
|
||||
from . import metrics, networks, limits, peerings, routers
|
||||
|
||||
|
||||
|
||||
|
||||
def get_firewalls_dict(config: dict):
|
||||
'''
|
||||
Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization.
|
||||
|
||||
Parameters:
|
||||
config (dict): The dict containing config like clients and limits
|
||||
Returns:
|
||||
firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules
|
||||
'''
|
||||
|
||||
firewalls_dict = defaultdict(int)
|
||||
read_mask = field_mask_pb2.FieldMask()
|
||||
read_mask.FromJsonString('name,versionedResources')
|
||||
|
||||
|
||||
response = config["clients"]["asset_client"].search_all_resources(
|
||||
request={
|
||||
"scope": f"organizations/{config['organization']}",
|
||||
"asset_types": ["compute.googleapis.com/Firewall"],
|
||||
"read_mask": read_mask,
|
||||
})
|
||||
for resource in response:
|
||||
project_id=re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", resource.name).group(2)
|
||||
network_name=""
|
||||
for versioned in resource.versioned_resources:
|
||||
for field_name, field_value in versioned.resource.items():
|
||||
if field_name == "network":
|
||||
network_name = re.search("[a-z0-9\-]*$", field_value).group(0)
|
||||
firewalls_dict[project_id]= defaultdict(int) if not project_id in firewalls_dict else firewalls_dict[project_id]
|
||||
firewalls_dict[project_id][network_name]=1 if not network_name in firewalls_dict[project_id] else firewalls_dict[project_id][network_name]+1
|
||||
break
|
||||
break
|
||||
return firewalls_dict
|
||||
|
||||
|
||||
|
||||
'''def get_firewalls_per_project_vpc(config, project_id ):
|
||||
|
||||
Returns returns the VPC firewall rules defined in a project
|
||||
|
||||
Parameters:
|
||||
config (dict): The dict containing config like clients and limits
|
||||
project_id (string): Project ID for the project containing the Cloud Router.
|
||||
Returns:
|
||||
sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project
|
||||
|
||||
get_firewalls_dict(config)
|
||||
sum_firewalls_per_vpc=dict()
|
||||
sum_firewalls=0
|
||||
page_token=None
|
||||
while (1):
|
||||
request = config["clients"]["discovery_client"].firewalls().list(
|
||||
project=project_id,pageToken=page_token)
|
||||
response = request.execute()
|
||||
sum_firewalls_per_vpc = dict()
|
||||
if 'items' in response:
|
||||
for firewall_rule in response['items']:
|
||||
sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1
|
||||
else:
|
||||
break
|
||||
page_token = response['pageToken'] if 'pageToken' in response else None
|
||||
if (not page_token):
|
||||
break
|
||||
|
||||
return sum_firewalls_per_vpc
|
||||
'''
|
||||
|
||||
def get_firewalls_data(config, metrics_dict,project_quotas_dict,firewalls_dict):
|
||||
'''
|
||||
Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric.
|
||||
|
||||
Parameters:
|
||||
config (dict): The dict containing config like clients and limits
|
||||
metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
|
||||
limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
|
||||
firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules
|
||||
Returns:
|
||||
None
|
||||
'''
|
||||
for project in config["monitored_projects"]:
|
||||
|
||||
current_quota_limit= project_quotas_dict[project]['global']["firewalls"]
|
||||
if current_quota_limit is None:
|
||||
print(
|
||||
f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas"
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
network_dict = networks.get_networks(config, project)
|
||||
|
||||
project_usage=0
|
||||
for net in network_dict:
|
||||
usage = 0
|
||||
if net['network_name'] in firewalls_dict:
|
||||
usage = firewalls_dict[project][net['network_name']]
|
||||
project_usage+=usage
|
||||
metrics.write_data_to_metric(
|
||||
config, project, usage, metrics_dict["metrics_per_project"]
|
||||
[f"firewalls"]["usage"]["name"],
|
||||
net['network_name'])
|
||||
|
||||
#firewall quotas are per project, not per single VPC
|
||||
metrics.write_data_to_metric(
|
||||
config, project, current_quota_limit['limit'], metrics_dict["metrics_per_project"]
|
||||
[f"firewalls"]["limit"]["name"],
|
||||
)
|
||||
metrics.write_data_to_metric(
|
||||
config, project, project_usage / current_quota_limit['limit'] if current_quota_limit['limit']!=0 else 0,
|
||||
metrics_dict["metrics_per_project"]
|
||||
[f"firewalls"]["utilization"]["name"],)
|
||||
|
||||
print(
|
||||
f"Wrote number of VPC Firewall Rules to metric for projects/{project}"
|
||||
)
|
Loading…
Reference in New Issue