zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pinning version for libs and terraform modules, adding docstrings, improving documentation of the code, tabs 2 spaces.

This commit is contained in:
Aurélien Legrand 2022-03-09 19:02:59 +01:00
parent 9f3ee4dc22
commit 221557d066
4 changed files with 778 additions and 561 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

349
examples/cloud-operations/network-dashboard/cloud-function/main.py
View File

@ -23,39 +23,43 @@ import google.api_core
import re
import random
#
monitored_projects_list = os.environ.get("monitored_projects_list").split(",") # list of projects from which function will get quotas information
monitoring_project_id = os.environ.get("monitoring_project_id") # project where the metrics and dahsboards will be created
monitoring_project_link = f"projects/{monitoring_project_id}"
service = discovery.build('compute', 'v1')
# DEFAULT LIMITS
limit_vpc_peer = os.environ.get("LIMIT_VPC_PEER").split(",") # 25
limit_l4 = os.environ.get("LIMIT_L4").split(",") # 75
limit_l7 = os.environ.get("LIMIT_L7").split(",") # 75
limit_instances = os.environ.get("LIMIT_INSTANCES").split(",") # ["default_value", "15000"]
limit_instances_ppg = os.environ.get("LIMIT_INSTANCES_PPG").split(",") # 15000
limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",") # 400
limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",") # 175
limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",") # 175
# DEFAULT LIMITS:
limit_vpc_peer = os.environ.get("LIMIT_VPC_PEER").split(",")
limit_l4 = os.environ.get("LIMIT_L4").split(",")
limit_l7 = os.environ.get("LIMIT_L7").split(",")
limit_instances = os.environ.get("LIMIT_INSTANCES").split(",")
limit_instances_ppg = os.environ.get("LIMIT_INSTANCES_PPG").split(",")
limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",")
limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",")
limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",")
# Cloud Function entry point
def quotas(request):
'''
Cloud Function Entry point, called by the scheduler.
Parameters:
request: for now, the Cloud Function is triggered by an HTTP trigger and this request correspond to the HTTP triggering request.
Returns:
'Function executed successfully'
'''
global client, interval
client, interval = create_client()
# Instances per VPC
instance_metric = create_gce_instances_metrics()
get_gce_instances_data(instance_metric)
# Number of VPC peerings per VPC
vpc_peering_active_metric, vpc_peering_metric = create_vpc_peering_metrics()
get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric)
# Internal L4 Forwarding Rules per VPC
forwarding_rules_metric = create_l4_forwarding_rules_metric()
get_l4_forwarding_rules_data(forwarding_rules_metric)
# Internal L4 Forwarding Rules per VPC peering group
# Existing GCP Monitoring metrics for L4 Forwarding Rules per Network
l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
@ -63,7 +67,6 @@ def quotas(request):
l4_forwarding_rules_ppg_metric = create_l4_forwarding_rules_ppg_metric()
get_pgg_data(l4_forwarding_rules_ppg_metric, l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg)
# Internal L7 Forwarding Rules per VPC peering group
# Existing GCP Monitoring metrics for L7 Forwarding Rules per Network
l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage"
l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit"
@ -71,7 +74,6 @@ def quotas(request):
l7_forwarding_rules_ppg_metric = create_l7_forwarding_rules_ppg_metric()
get_pgg_data(l7_forwarding_rules_ppg_metric, l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg)
# Subnet ranges per VPC peering group
# Existing GCP Monitoring metrics for Subnet Ranges per Network
subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage"
subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
@ -79,7 +81,6 @@ def quotas(request):
subnet_ranges_ppg_metric = create_subnet_ranges_ppg_metric()
get_pgg_data(subnet_ranges_ppg_metric, subnet_ranges_usage, subnet_ranges_limit, limit_subnets)
# GCE Instances per VPC peering group
# Existing GCP Monitoring metrics for GCE per Network
gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
@ -90,6 +91,15 @@ def quotas(request):
return 'Function executed successfully'
def create_client():
'''
Creates the monitoring API client, that will be used to create, read and update custom metrics.
Parameters:
None
Returns:
client (monitoring_v3.MetricServiceClient): Monitoring API client
interval (monitoring_v3.TimeInterval): Interval for the metric data points (24 hours)
'''
try:
client = monitoring_v3.MetricServiceClient()
now = time.time()
@ -104,9 +114,15 @@ def create_client():
except Exception as e:
raise Exception("Error occurred creating the client: {}".format(e))
# Creates usage, limit and utilization Cloud monitoring metrics for GCE instances
# Returns a dictionary with the names and descriptions of the created metrics
def create_gce_instances_metrics():
'''
Creates a dictionary with the name and description of 3 metrics for GCE instances per VPC network: usage, limit and utilization.
Parameters:
None
Returns:
instance_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
instance_metric = {}
instance_metric["usage_name"] = "number_of_instances_usage"
instance_metric["limit_name"] = "number_of_instances_limit"
@ -122,8 +138,17 @@ def create_gce_instances_metrics():
return instance_metric
# Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
def create_metric(metric_name, description):
'''
Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
Parameters:
metric_name (string): Name of the metric to be created
description (string): Description of the metric to be created
Returns:
None
'''
client = monitoring_v3.MetricServiceClient()
metric_link = f"custom.googleapis.com/{metric_name}"
@ -131,8 +156,7 @@ def create_metric(metric_name, description):
for desc in client.list_metric_descriptors(name=monitoring_project_link):
types.append(desc.type)
# If the metric doesn't exist yet, then we create it
if metric_link not in types:
if metric_link not in types: # If the metric doesn't exist yet, then we create it
descriptor = ga_metric.MetricDescriptor()
descriptor.type = f"custom.googleapis.com/{metric_name}"
descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE
@ -142,7 +166,14 @@ def create_metric(metric_name, description):
print("Created {}.".format(descriptor.name))
def get_gce_instances_data(instance_metric):
'''
Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric.
Parameters:
instance_metric (dictionary of string: string): metrics name and description for GCE instances per VPC Network
Returns:
None
'''
# Existing GCP Monitoring metrics for GCE instances
metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
@ -165,8 +196,16 @@ def get_gce_instances_data(instance_metric):
print(f"Wrote number of instances to metric for projects/{project}")
# Creates 2 metrics: VPC peerings per VPC and active VPC peerings per VPC
def create_vpc_peering_metrics():
'''
Creates 2 dictionaries with the name and description of 3 metrics for Active VPC peering and All VPC peerings: usage, limit and utilization.
Parameters:
None
Returns:
vpc_peering_active_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
vpc_peering_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
vpc_peering_active_metric = {}
vpc_peering_active_metric["usage_name"] = "number_of_active_vpc_peerings_usage"
vpc_peering_active_metric["limit_name"] = "number_of_active_vpc_peerings_limit"
@ -195,24 +234,40 @@ def create_vpc_peering_metrics():
return vpc_peering_active_metric, vpc_peering_metric
# Populates data for VPC peerings per VPC and active VPC peerings per VPC
def get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric):
'''
Gets the data for VPC peerings (active or not) and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
Parameters:
vpc_peering_active_metric (dictionary of string: string):
Returns:
None
'''
for project in monitored_projects_list:
active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_vpc_peer)
for peering in active_vpc_peerings:
write_data_to_metric(project, peering['active_peerings'], vpc_peering_active_metric["usage_name"], peering['network name'])
write_data_to_metric(project, peering['network_limit'], vpc_peering_active_metric["limit_name"], peering['network name'])
write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], vpc_peering_active_metric["utilization_name"], peering['network name'])
write_data_to_metric(project, peering['active_peerings'], vpc_peering_active_metric["usage_name"], peering['network_name'])
write_data_to_metric(project, peering['network_limit'], vpc_peering_active_metric["limit_name"], peering['network_name'])
write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], vpc_peering_active_metric["utilization_name"], peering['network_name'])
print("Wrote number of active VPC peerings to custom metric for project:", project)
for peering in vpc_peerings:
write_data_to_metric(project, peering['peerings'], vpc_peering_metric["usage_name"], peering['network name'])
write_data_to_metric(project, peering['network_limit'], vpc_peering_metric["limit_name"], peering['network name'])
write_data_to_metric(project, peering['peerings'] / peering['network_limit'], vpc_peering_metric["utilization_name"], peering['network name'])
write_data_to_metric(project, peering['peerings'], vpc_peering_metric["usage_name"], peering['network_name'])
write_data_to_metric(project, peering['network_limit'], vpc_peering_metric["limit_name"], peering['network_name'])
write_data_to_metric(project, peering['peerings'] / peering['network_limit'], vpc_peering_metric["utilization_name"], peering['network_name'])
print("Wrote number of VPC peerings to custom metric for project:", project)
# gathers number of VPC peerings, active VPC peerings and limits, returns 2 dictionaries: active_vpc_peerings and vpc_peerings (including inactive and active ones)
def gather_vpc_peerings_data(project_id, limit_list):
'''
Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
Parameters:
project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs.
limit_list (list of string): Used to get the limit per VPC or the default limit.
Returns:
active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering.
peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering.
'''
active_peerings_dict = []
peerings_dict = []
request = service.networks().list(project=project_id)
@ -231,15 +286,23 @@ def gather_vpc_peerings_data(project_id, limit_list):
peerings_count = 0
active_peerings_count = 0
active_d = {'project_id': project_id,'network name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)}
active_d = {'project_id': project_id,'network_name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)}
active_peerings_dict.append(active_d)
d = {'project_id': project_id,'network name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)}
d = {'project_id': project_id,'network_name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)}
peerings_dict.append(d)
return active_peerings_dict, peerings_dict
# Check if the VPC has a specific limit for a specific metric, if so, returns that limit, if not, returns the default limit
def get_limit(network_name, limit_list):
'''
Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
Parameters:
network_name (string): Name of the VPC network.
limit_list (list of string): Used to get the limit per VPC or the default limit.
Returns:
limit (int): Limit for that VPC and that metric.
'''
if network_name in limit_list:
return int(limit_list[limit_list.index(network_name) + 1])
else:
@ -248,8 +311,15 @@ def get_limit(network_name, limit_list):
else:
return 0
# Creates the custom metrics for L4 internal forwarding Rules
def create_l4_forwarding_rules_metric():
'''
Creates a dictionary with the name and description of 3 metrics for L4 Internal Forwarding Rules per VPC network: usage, limit and utilization.
Parameters:
None
Returns:
forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
forwarding_rules_metric = {}
forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_usage"
forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_limit"
@ -266,7 +336,14 @@ def create_l4_forwarding_rules_metric():
return forwarding_rules_metric
def get_l4_forwarding_rules_data(forwarding_rules_metric):
'''
Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
Parameters:
forwarding_rules_metric (dictionary of string: string): metrics name and description for L4 Internal Forwarding Rules per VPC Network.
Returns:
None
'''
# Existing GCP Monitoring metrics for L4 Forwarding Rules
l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
@ -288,8 +365,15 @@ def get_l4_forwarding_rules_data(forwarding_rules_metric):
print(f"Wrote number of L4 forwarding rules to metric for projects/{project}")
# Creates the custom metrics for L4 internal forwarding Rules per VPC Peering Group
def create_l4_forwarding_rules_ppg_metric():
'''
Creates a dictionary with the name and description of 3 metrics for L4 Internal Forwarding Rules per VPC Peering Group: usage, limit and utilization.
Parameters:
None
Returns:
forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
forwarding_rules_metric = {}
forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_ppg_usage"
forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_ppg_limit"
@ -305,8 +389,15 @@ def create_l4_forwarding_rules_ppg_metric():
return forwarding_rules_metric
# Creates the custom metrics for L7 internal forwarding Rules per VPC Peering Group
def create_l7_forwarding_rules_ppg_metric():
'''
Creates a dictionary with the name and description of 3 metrics for L7 Internal Forwarding Rules per VPC Peering Group: usage, limit and utilization.
Parameters:
None
Returns:
forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
forwarding_rules_metric = {}
forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l7_ppg_usage"
forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l7_ppg_limit"
@ -323,6 +414,14 @@ def create_l7_forwarding_rules_ppg_metric():
return forwarding_rules_metric
def create_subnet_ranges_ppg_metric():
'''
Creates a dictionary with the name and description of 3 metrics for Subnet Ranges per VPC Peering Group: usage, limit and utilization.
Parameters:
None
Returns:
metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
metric = {}
metric["usage_name"] = "number_of_subnet_IP_ranges_usage"
@ -340,6 +439,14 @@ def create_subnet_ranges_ppg_metric():
return metric
def create_gce_instances_ppg_metric():
'''
Creates a dictionary with the name and description of 3 metrics for GCE Instances per VPC Peering Group: usage, limit and utilization.
Parameters:
None
Returns:
metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
'''
metric = {}
metric["usage_name"] = "number_of_instances_ppg_usage"
@ -356,9 +463,18 @@ def create_gce_instances_ppg_metric():
return metric
# Populates data for the custom metrics for L4 internal forwarding Rules per Peering Group
def get_pgg_data(forwarding_rules_ppg_metric, usage_metric, limit_metric, limit_ppg):
def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
'''
This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored.
Parameters:
metric_dict (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description)
usage_metric (string): Name of the existing GCP metric for usage per VPC network.
limit_metric (string): Name of the existing GCP metric for limit per VPC network.
limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit).
Returns:
None
'''
for project in monitored_projects_list:
network_dict_list = gather_peering_data(project)
# Network dict list is a list of dictionary (one for each network)
@ -390,12 +506,24 @@ def get_pgg_data(forwarding_rules_ppg_metric, usage_metric, limit_metric, limit_
peered_network["usage"] = usage
peered_network["limit"] = limit
count_effective_limit(project, network_dict, forwarding_rules_ppg_metric["usage_name"], forwarding_rules_ppg_metric["limit_name"], forwarding_rules_ppg_metric["utilization_name"], limit_ppg)
print(f"Wrote {forwarding_rules_ppg_metric['usage_name']} to metric for peering group {network_dict['network_name']} in {project}")
count_effective_limit(project, network_dict, metric_dict["usage_name"], metric_dict["limit_name"], metric_dict["utilization_name"], limit_ppg)
print(f"Wrote {metric_dict['usage_name']} to metric for peering group {network_dict['network_name']} in {project}")
# Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to metric
# https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg):
'''
Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
Parameters:
project_id (string): Project ID for the project to be analyzed.
network_dict (dictionary of string: string): Contains all required information about the network to get the usage, limit and utilization.
usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group.
limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group.
utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group.
limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit).
Returns:
None
'''
if network_dict['peerings'] == []:
return
@ -424,8 +552,15 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met
write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name'])
write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name'])
# Takes a project id as argument (and service for the GCP API call) and return a dictionary with the list of networks
def get_networks(project_id):
'''
Returns a dictionary of all networks in a project.
Parameters:
project_id (string): Project ID for the project containing the networks.
Returns:
network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s)
'''
request = service.networks().list(project=project_id)
response = request.execute()
network_dict = []
@ -437,12 +572,18 @@ def get_networks(project_id):
network_dict.append(d)
return network_dict
# gathers data for peered networks for the given project_id
def gather_peering_data(project_id):
'''
Returns a dictionary of all peerings for all networks in a project.
Parameters:
project_id (string): Project ID for the project containing the networks.
Returns:
network_list (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) of peered networks.
'''
request = service.networks().list(project=project_id)
response = request.execute()
# list of networks in that project
network_list = []
if 'items' in response:
for network in response['items']:
@ -461,6 +602,15 @@ def gather_peering_data(project_id):
return network_list
def get_network_id(project_id, network_name):
'''
Returns the network_id for a specific project / network name.
Parameters:
project_id (string): Project ID for the project containing the networks.
network_name (string): Name of the network
Returns:
network_id (int): Network ID.
'''
request = service.networks().list(project=project_id)
response = request.execute()
@ -477,30 +627,53 @@ def get_network_id(project_id, network_name):
return network_id
# retrieves quota for "type" in project_link, otherwise returns null (assume 0 for building comparison vs limits)
def get_quota_current_usage(project_link, type):
def get_quota_current_usage(project_link, metric_name):
'''
Retrieves quota usage for a specific metric.
Parameters:
project_link (string): Project link.
metric_name (string): Name of the metric.
Returns:
results_list (list of string): Current usage.
'''
results = client.list_time_series(request={
"name": project_link,
"filter": f'metric.type = "{type}"',
"filter": f'metric.type = "{metric_name}"',
"interval": interval,
"view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL
})
results_list = list(results)
return (results_list)
# retrieves quota for services limits
def get_quota_current_limit(project_link, type):
def get_quota_current_limit(project_link, metric_name):
'''
Retrieves limit for a specific metric.
Parameters:
project_link (string): Project link.
metric_name (string): Name of the metric.
Returns:
results_list (list of string): Current limit.
'''
results = client.list_time_series(request={
"name": project_link,
"filter": f'metric.type = "{type}"',
"filter": f'metric.type = "{metric_name}"',
"interval": interval,
"view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL
})
results_list = list(results)
return (results_list)
return results_list
# Customize the quota output
def customize_quota_view(quota_results):
'''
Customize the quota output for an easier parsable output.
Parameters:
quota_results (string): Input from get_quota_current_usage or get_quota_current_limit. Contains the Current usage or limit for all networks in that project.
Returns:
quotaViewList (list of dictionaries of string: string): Current quota usage or limit.
'''
quotaViewList = []
for result in quota_results:
quotaViewJson = {}
@ -509,38 +682,59 @@ def customize_quota_view(quota_results):
for val in result.points:
quotaViewJson.update({'value': val.value.int64_value})
quotaViewList.append(quotaViewJson)
return (quotaViewList)
return quotaViewList
# Takes a network dictionary and updates it with the quota usage and limits values
def set_usage_limits(network, quota_usage, quota_limit, limit_list):
def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
'''
Updates the network dictionary with quota usage and limit values.
Parameters:
network_dict (dictionary of string: string): Contains network information.
quota_usage (list of dictionaries of string: string): Current quota usage.
quota_limit (list of dictionaries of string: string): Current quota limit.
limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit).
Returns:
None
'''
if quota_usage:
for net in quota_usage:
if net['network_id'] == network['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
network['usage'] = net['value'] # set network usage in dictionary
if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
network_dict['usage'] = net['value'] # set network usage in dictionary
break
else:
network['usage'] = 0 # if network does not appear in GCP quotas
network_dict['usage'] = 0 # if network does not appear in GCP quotas
else:
network['usage'] = 0 # if quotas does not appear in GCP quotas
network_dict['usage'] = 0 # if quotas does not appear in GCP quotas
if quota_limit:
for net in quota_limit:
if net['network_id'] == network['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
network['limit'] = net['value'] # set network limit in dictionary
if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
network_dict['limit'] = net['value'] # set network limit in dictionary
break
else:
if network['network name'] in limit_list: # if network limit is in the environmental variables
network['limit'] = int(limit_list[limit_list.index(network['network name']) + 1])
if network_dict['network name'] in limit_list: # if network limit is in the environmental variables
network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1])
else:
network['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value
network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value
else: # if quotas does not appear in GCP quotas
if network['network name'] in limit_list:
network['limit'] = int(limit_list[limit_list.index(network['network name']) + 1]) # ["default", 100, "networkname", 200]
if network_dict['network name'] in limit_list:
network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) # ["default", 100, "networkname", 200]
else:
network['limit'] = int(limit_list[limit_list.index('default_value') + 1])
network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1])
# Takes a network dictionary (with at least network_id and network_name) and returns usage and limit for that network
def get_usage_limit(network, quota_usage, quota_limit, limit_list):
'''
Returns usage and limit for a specific network and metric.
Parameters:
network_dict (dictionary of string: string): Contains network information.
quota_usage (list of dictionaries of string: string): Current quota usage for all networks in that project.
quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project.
limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit).
Returns:
usage (int): Current usage for that network.
limit (int): Current usage for that network.
'''
usage = 0
limit = 0
@ -568,10 +762,19 @@ def get_usage_limit(network, quota_usage, quota_limit, limit_list):
return usage, limit
# Writes data to Cloud Monitoring data
# Note that the monitoring_project_id here should be the monitoring project where the metrics are written
# and monitored_project_id is the monitored project, containing the network and resources
def write_data_to_metric(monitored_project_id, value, metric_name, network_name):
'''
Writes data to Cloud Monitoring custom metrics.
Parameters:
monitored_project_id: ID of the project where the resource lives (will be added as a label)
value (int): Value for the data point of the metric.
metric_name (string): Name of the metric
network_name (string): Name of the network (will be added as a label)
Returns:
usage (int): Current usage for that network.
limit (int): Current usage for that network.
'''
series = monitoring_v3.TimeSeries()
series.metric.type = f"custom.googleapis.com/{metric_name}"
series.resource.type = "global"

16
examples/cloud-operations/network-dashboard/cloud-function/requirements.txt
View File

@ -1,8 +1,8 @@
regex
google-api-python-client
google-auth
google-auth-httplib2
google-cloud-logging
google-cloud-monitoring
oauth2client
google-api-core
regex==2022.3.2
google-api-python-client==2.39.0
google-auth==2.6.0
google-auth-httplib2==0.1.0
google-cloud-logging==3.0.0
google-cloud-monitoring==2.9.1
oauth2client==4.1.3
google-api-core==2.7.0

4
examples/cloud-operations/network-dashboard/main.tf
View File

@ -41,7 +41,7 @@ locals {
################################################
module "project-monitoring" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
name = "monitoring"
parent = "organizations/${var.organization_id}"
prefix = var.prefix
@ -54,7 +54,7 @@ module "project-monitoring" {
################################################
module "service-account-function" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/iam-service-account"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/iam-service-account?ref=v14.0.0"
project_id = module.project-monitoring.project_id
name = "sa-dash"
generate_key = false

42
examples/cloud-operations/network-dashboard/tests/test.tf
View File

@ -1,4 +1,18 @@
# Creating test infrastructure
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
resource "google_folder" "test-net-dash" {
display_name = "test-net-dash"
@ -8,7 +22,7 @@ resource "google_folder" "test-net-dash" {
##### Creating host projects, VPCs, service projects #####
module "project-hub" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
name = "test-host-hub"
parent = google_folder.test-net-dash.name
prefix = var.prefix
@ -22,7 +36,7 @@ module "project-hub" {
}
module "vpc-hub" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0"
project_id = module.project-hub.project_id
name = "vpc-hub"
subnets = [
@ -36,7 +50,7 @@ module "vpc-hub" {
}
module "project-svc-hub" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
parent = google_folder.test-net-dash.name
billing_account = var.billing_account
prefix = var.prefix
@ -50,7 +64,7 @@ module "project-svc-hub" {
}
module "project-prod" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
name = "test-host-prod"
parent = google_folder.test-net-dash.name
prefix = var.prefix
@ -64,7 +78,7 @@ module "project-prod" {
}
module "vpc-prod" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0"
project_id = module.project-prod.project_id
name = "vpc-prod"
subnets = [
@ -78,7 +92,7 @@ module "vpc-prod" {
}
module "project-svc-prod" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
parent = google_folder.test-net-dash.name
billing_account = var.billing_account
prefix = var.prefix
@ -92,7 +106,7 @@ module "project-svc-prod" {
}
module "project-dev" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
name = "test-host-dev"
parent = google_folder.test-net-dash.name
prefix = var.prefix
@ -106,7 +120,7 @@ module "project-dev" {
}
module "vpc-dev" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0"
project_id = module.project-dev.project_id
name = "vpc-dev"
subnets = [
@ -120,7 +134,7 @@ module "vpc-dev" {
}
module "project-svc-dev" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0"
parent = google_folder.test-net-dash.name
billing_account = var.billing_account
prefix = var.prefix
@ -136,26 +150,26 @@ module "project-svc-dev" {
##### Creating VPC peerings #####
module "hub-to-prod-peering" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering"
local_network = module.vpc-hub.self_link
peer_network = module.vpc-prod.self_link
}
module "prod-to-hub-peering" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering"
local_network = module.vpc-prod.self_link
peer_network = module.vpc-hub.self_link
depends_on = [module.hub-to-prod-peering]
}
module "hub-to-dev-peering" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering"
local_network = module.vpc-hub.self_link
peer_network = module.vpc-dev.self_link
}
module "dev-to-hub-peering" {
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering"
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering"
local_network = module.vpc-dev.self_link
peer_network = module.vpc-hub.self_link
depends_on = [module.hub-to-dev-peering]