zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactoring main.py

This commit is contained in:
Aurélien Legrand 2022-03-17 20:08:58 +01:00 committed by Julio Castillo
parent 7802c33988
commit 7874f9c02b
2 changed files with 270 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

322
examples/cloud-operations/network-dashboard/cloud-function/main.py
View File

@ -14,15 +14,12 @@
# limitations under the License. # limitations under the License.
# #
import time
import os
import yaml
from google.cloud import monitoring_v3 from google.cloud import monitoring_v3
from googleapiclient import discovery from googleapiclient import discovery
from google.api import metric_pb2 as ga_metric from google.api import metric_pb2 as ga_metric
import time
import os
import google.api_core
import re
import random
import yaml
# list of projects from which function will get quotas information # list of projects from which function will get quotas information
MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",") MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",")
@ -41,6 +38,17 @@ LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",")
LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",") LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",")
LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",") LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",")
# Existing GCP metrics per network
L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage"
L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit"
SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage"
SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
def main(event, context): def main(event, context):
''' '''
Cloud Function Entry point, called by the scheduler. Cloud Function Entry point, called by the scheduler.
@ -59,32 +67,31 @@ def main(event, context):
get_vpc_peering_data(metrics_dict) get_vpc_peering_data(metrics_dict)
get_l4_forwarding_rules_data(metrics_dict) get_l4_forwarding_rules_data(metrics_dict)
# Existing GCP Monitoring metrics for L4 Forwarding Rules per Network get_pgg_data(
l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" metrics_dict["metrics_per_peering_group"]
l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" ["l4_forwarding_rules_per_peering_group"],
L4_FORWARDING_RULES_USAGE_METRIC, L4_FORWARDING_RULES_LIMIT_METRIC,
LIMIT_L4_PPG)
get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, LIMIT_L4_PPG) get_pgg_data(
metrics_dict["metrics_per_peering_group"]
["l7_forwarding_rules_per_peering_group"],
L7_FORWARDING_RULES_USAGE_METRIC, L7_FORWARDING_RULES_LIMIT_METRIC,
LIMIT_L7_PPG)
# Existing GCP Monitoring metrics for L7 Forwarding Rules per Network get_pgg_data(
l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" metrics_dict["metrics_per_peering_group"]
l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" ["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC,
SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS)
get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, LIMIT_L7_PPG) get_pgg_data(
metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"],
# Existing GCP Monitoring metrics for Subnet Ranges per Network GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC,
subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" LIMIT_INSTANCES_PPG)
subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, LIMIT_SUBNETS)
# Existing GCP Monitoring metrics for GCE per Network
gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, LIMIT_INSTANCES_PPG)
return 'Function executed successfully' return 'Function executed successfully'
def create_client(): def create_client():
''' '''
Creates the monitoring API client, that will be used to create, read and update custom metrics. Creates the monitoring API client, that will be used to create, read and update custom metrics.
@ -100,15 +107,21 @@ def create_client():
now = time.time() now = time.time()
seconds = int(now) seconds = int(now)
nanos = int((now - seconds) * 10**9) nanos = int((now - seconds) * 10**9)
interval = monitoring_v3.TimeInterval( interval = monitoring_v3.TimeInterval({
{ "end_time": {
"end_time": {"seconds": seconds, "nanos": nanos}, "seconds": seconds,
"start_time": {"seconds": (seconds - 86400), "nanos": nanos}, "nanos": nanos
},
"start_time": {
"seconds": (seconds - 86400),
"nanos": nanos
},
}) })
return (client, interval) return (client, interval)
except Exception as e: except Exception as e:
raise Exception("Error occurred creating the client: {}".format(e)) raise Exception("Error occurred creating the client: {}".format(e))
def create_metrics(): def create_metrics():
with open("metrics.yaml", 'r') as stream: with open("metrics.yaml", 'r') as stream:
try: try:
@ -123,6 +136,7 @@ def create_metrics():
except yaml.YAMLError as exc: except yaml.YAMLError as exc:
print(exc) print(exc)
def create_metric(metric_name, description): def create_metric(metric_name, description):
''' '''
Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
@ -147,9 +161,11 @@ def create_metric(metric_name, description):
descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE
descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE
descriptor.description = description descriptor.description = description
descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, metric_descriptor=descriptor) descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK,
metric_descriptor=descriptor)
print("Created {}.".format(descriptor.name)) print("Created {}.".format(descriptor.name))
def get_gce_instances_data(metrics_dict): def get_gce_instances_data(metrics_dict):
''' '''
Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric.
@ -166,17 +182,27 @@ def get_gce_instances_data(metrics_dict):
for project in MONITORED_PROJECTS_LIST: for project in MONITORED_PROJECTS_LIST:
network_dict = get_networks(project) network_dict = get_networks(project)
current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) current_quota_usage = get_quota_current_usage(f"projects/{project}",
current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) metric_instances_usage)
current_quota_limit = get_quota_current_limit(f"projects/{project}",
metric_instances_limit)
current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_usage_view = customize_quota_view(current_quota_usage)
current_quota_limit_view = customize_quota_view(current_quota_limit) current_quota_limit_view = customize_quota_view(current_quota_limit)
for net in network_dict: for net in network_dict:
set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_INSTANCES) set_usage_limits(net, current_quota_usage_view, current_quota_limit_view,
write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["instance_per_network"]["usage"]["name"], net['network name']) LIMIT_INSTANCES)
write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["limit"]["name"], net['network name']) write_data_to_metric(
write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["utilization"]["name"], net['network name']) project, net['usage'], metrics_dict["metrics_per_network"]
["instance_per_network"]["usage"]["name"], net['network name'])
write_data_to_metric(
project, net['limit'], metrics_dict["metrics_per_network"]
["instance_per_network"]["limit"]["name"], net['network name'])
write_data_to_metric(
project, net['usage'] / net['limit'],
metrics_dict["metrics_per_network"]["instance_per_network"]
["utilization"]["name"], net['network name'])
print(f"Wrote number of instances to metric for projects/{project}") print(f"Wrote number of instances to metric for projects/{project}")
@ -191,19 +217,38 @@ def get_vpc_peering_data(metrics_dict):
None None
''' '''
for project in MONITORED_PROJECTS_LIST: for project in MONITORED_PROJECTS_LIST:
active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, LIMIT_VPC_PEER) active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(
project, LIMIT_VPC_PEER)
for peering in active_vpc_peerings: for peering in active_vpc_peerings:
write_data_to_metric(project, peering['active_peerings'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["usage"]["name"], peering['network_name']) write_data_to_metric(
write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["limit"]["name"], peering['network_name']) project, peering['active_peerings'],
write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["utilization"]["name"], peering['network_name']) metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
print("Wrote number of active VPC peerings to custom metric for project:", project) ["usage"]["name"], peering['network_name'])
write_data_to_metric(
project, peering['network_limit'], metrics_dict["metrics_per_network"]
["vpc_peering_active_per_network"]["limit"]["name"],
peering['network_name'])
write_data_to_metric(
project, peering['active_peerings'] / peering['network_limit'],
metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
["utilization"]["name"], peering['network_name'])
print("Wrote number of active VPC peerings to custom metric for project:",
project)
for peering in vpc_peerings: for peering in vpc_peerings:
write_data_to_metric(project, peering['peerings'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["usage"]["name"], peering['network_name']) write_data_to_metric(
write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["limit"]["name"], peering['network_name']) project, peering['peerings'], metrics_dict["metrics_per_network"]
write_data_to_metric(project, peering['peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["utilization"]["name"], peering['network_name']) ["vpc_peering_per_network"]["usage"]["name"], peering['network_name'])
write_data_to_metric(
project, peering['network_limit'], metrics_dict["metrics_per_network"]
["vpc_peering_per_network"]["limit"]["name"], peering['network_name'])
write_data_to_metric(
project, peering['peerings'] / peering['network_limit'],
metrics_dict["metrics_per_network"]["vpc_peering_per_network"]
["utilization"]["name"], peering['network_name'])
print("Wrote number of VPC peerings to custom metric for project:", project) print("Wrote number of VPC peerings to custom metric for project:", project)
def gather_vpc_peerings_data(project_id, limit_list): def gather_vpc_peerings_data(project_id, limit_list):
''' '''
Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
@ -233,13 +278,24 @@ def gather_vpc_peerings_data(project_id, limit_list):
peerings_count = 0 peerings_count = 0
active_peerings_count = 0 active_peerings_count = 0
active_d = {'project_id': project_id,'network_name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)} active_d = {
'project_id': project_id,
'network_name': network['name'],
'active_peerings': active_peerings_count,
'network_limit': get_limit(network['name'], limit_list)
}
active_peerings_dict.append(active_d) active_peerings_dict.append(active_d)
d = {'project_id': project_id,'network_name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)} d = {
'project_id': project_id,
'network_name': network['name'],
'peerings': peerings_count,
'network_limit': get_limit(network['name'], limit_list)
}
peerings_dict.append(d) peerings_dict.append(d)
return active_peerings_dict, peerings_dict return active_peerings_dict, peerings_dict
def get_limit(network_name, limit_list): def get_limit(network_name, limit_list):
''' '''
Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
@ -258,6 +314,7 @@ def get_limit(network_name, limit_list):
else: else:
return 0 return 0
def get_l4_forwarding_rules_data(metrics_dict): def get_l4_forwarding_rules_data(metrics_dict):
''' '''
Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
@ -274,19 +331,33 @@ def get_l4_forwarding_rules_data(metrics_dict):
for project in MONITORED_PROJECTS_LIST: for project in MONITORED_PROJECTS_LIST:
network_dict = get_networks(project) network_dict = get_networks(project)
current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) current_quota_usage = get_quota_current_usage(f"projects/{project}",
current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) l4_forwarding_rules_usage)
current_quota_limit = get_quota_current_limit(f"projects/{project}",
l4_forwarding_rules_limit)
current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_usage_view = customize_quota_view(current_quota_usage)
current_quota_limit_view = customize_quota_view(current_quota_limit) current_quota_limit_view = customize_quota_view(current_quota_limit)
for net in network_dict: for net in network_dict:
set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_L4) set_usage_limits(net, current_quota_usage_view, current_quota_limit_view,
write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name']) LIMIT_L4)
write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name']) write_data_to_metric(
write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["utilization"]["name"], net['network name']) project, net['usage'], metrics_dict["metrics_per_network"]
["l4_forwarding_rules_per_network"]["usage"]["name"],
net['network name'])
write_data_to_metric(
project, net['limit'], metrics_dict["metrics_per_network"]
["l4_forwarding_rules_per_network"]["limit"]["name"],
net['network name'])
write_data_to_metric(
project, net['usage'] / net['limit'],
metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]
["utilization"]["name"], net['network name'])
print(
f"Wrote number of L4 forwarding rules to metric for projects/{project}")
print(f"Wrote number of L4 forwarding rules to metric for projects/{project}")
def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
''' '''
@ -310,31 +381,46 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
# For each network in this GCP project # For each network in this GCP project
for network_dict in network_dict_list: for network_dict in network_dict_list:
current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric) current_quota_usage = get_quota_current_usage(f"projects/{project}",
current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) usage_metric)
current_quota_limit = get_quota_current_limit(f"projects/{project}",
limit_metric)
current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_usage_view = customize_quota_view(current_quota_usage)
current_quota_limit_view = customize_quota_view(current_quota_limit) current_quota_limit_view = customize_quota_view(current_quota_limit)
usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg) usage, limit = get_usage_limit(network_dict, current_quota_usage_view,
current_quota_limit_view, limit_ppg)
# Here we add usage and limit to the network dictionary # Here we add usage and limit to the network dictionary
network_dict["usage"] = usage network_dict["usage"] = usage
network_dict["limit"] = limit network_dict["limit"] = limit
# For every peered network, get usage and limits # For every peered network, get usage and limits
for peered_network in network_dict['peerings']: for peered_network in network_dict['peerings']:
peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric)) peering_project_usage = customize_quota_view(
peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) get_quota_current_usage(f"projects/{peered_network['project_id']}",
usage_metric))
peering_project_limit = customize_quota_view(
get_quota_current_limit(f"projects/{peered_network['project_id']}",
limit_metric))
usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg) usage, limit = get_usage_limit(peered_network, peering_project_usage,
peering_project_limit, limit_ppg)
# Here we add usage and limit to the peered network dictionary # Here we add usage and limit to the peered network dictionary
peered_network["usage"] = usage peered_network["usage"] = usage
peered_network["limit"] = limit peered_network["limit"] = limit
count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_ppg) count_effective_limit(project, network_dict, metric_dict["usage"]["name"],
print(f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}") metric_dict["limit"]["name"],
metric_dict["utilization"]["name"], limit_ppg)
print(
f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}"
)
def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg):
def count_effective_limit(project_id, network_dict, usage_metric_name,
limit_metric_name, utilization_metric_name,
limit_ppg):
''' '''
Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
@ -359,12 +445,15 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met
peering_group_usage += peered_network['usage'] peering_group_usage += peered_network['usage']
# Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit)
limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg)) limit_step1 = max(network_dict['limit'],
get_limit(network_dict['network_name'], limit_ppg))
# Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network
limit_step2 = [] limit_step2 = []
for peered_network in network_dict['peerings']: for peered_network in network_dict['peerings']:
limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg))) limit_step2.append(
max(peered_network['limit'],
get_limit(peered_network['network_name'], limit_ppg)))
# Calculates effective limit: Step 3: Find minimum from the list created by Step 2 # Calculates effective limit: Step 3: Find minimum from the list created by Step 2
limit_step3 = min(limit_step2) limit_step3 = min(limit_step2)
@ -373,9 +462,13 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met
effective_limit = max(limit_step1, limit_step3) effective_limit = max(limit_step1, limit_step3)
utilization = peering_group_usage / effective_limit utilization = peering_group_usage / effective_limit
write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name']) write_data_to_metric(project_id, peering_group_usage, usage_metric_name,
write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name']) network_dict['network_name'])
write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) write_data_to_metric(project_id, effective_limit, limit_metric_name,
network_dict['network_name'])
write_data_to_metric(project_id, utilization, utilization_metric_name,
network_dict['network_name'])
def get_networks(project_id): def get_networks(project_id):
''' '''
@ -397,6 +490,7 @@ def get_networks(project_id):
network_dict.append(d) network_dict.append(d)
return network_dict return network_dict
def gather_peering_data(project_id): def gather_peering_data(project_id):
''' '''
Returns a dictionary of all peerings for all networks in a project. Returns a dictionary of all peerings for all networks in a project.
@ -412,20 +506,36 @@ def gather_peering_data(project_id):
network_list = [] network_list = []
if 'items' in response: if 'items' in response:
for network in response['items']: for network in response['items']:
net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]} net = {
'project_id': project_id,
'network_name': network['name'],
'network_id': network['id'],
'peerings': []
}
if 'peerings' in network: if 'peerings' in network:
STATE = network['peerings'][0]['state'] STATE = network['peerings'][0]['state']
if STATE == "ACTIVE": if STATE == "ACTIVE":
for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}" for peered_network in network[
start = peered_network['network'].find("projects/") + len('projects/') 'peerings']: # "projects/{project_name}/global/networks/{network_name}"
start = peered_network['network'].find("projects/") + len(
'projects/')
end = peered_network['network'].find("/global") end = peered_network['network'].find("/global")
peered_project = peered_network['network'][start:end] peered_project = peered_network['network'][start:end]
peered_network_name = peered_network['network'].split("networks/")[1] peered_network_name = peered_network['network'].split(
peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)} "networks/")[1]
peered_net = {
'project_id':
peered_project,
'network_name':
peered_network_name,
'network_id':
get_network_id(peered_project, peered_network_name)
}
net["peerings"].append(peered_net) net["peerings"].append(peered_net)
network_list.append(net) network_list.append(net)
return network_list return network_list
def get_network_id(project_id, network_name): def get_network_id(project_id, network_name):
''' '''
Returns the network_id for a specific project / network name. Returns the network_id for a specific project / network name.
@ -452,6 +562,7 @@ def get_network_id(project_id, network_name):
return network_id return network_id
def get_quota_current_usage(project_link, metric_name): def get_quota_current_usage(project_link, metric_name):
''' '''
Retrieves quota usage for a specific metric. Retrieves quota usage for a specific metric.
@ -464,7 +575,8 @@ def get_quota_current_usage(project_link, metric_name):
''' '''
client, interval = create_client() client, interval = create_client()
results = client.list_time_series(request={ results = client.list_time_series(
request={
"name": project_link, "name": project_link,
"filter": f'metric.type = "{metric_name}"', "filter": f'metric.type = "{metric_name}"',
"interval": interval, "interval": interval,
@ -473,6 +585,7 @@ def get_quota_current_usage(project_link, metric_name):
results_list = list(results) results_list = list(results)
return (results_list) return (results_list)
def get_quota_current_limit(project_link, metric_name): def get_quota_current_limit(project_link, metric_name):
''' '''
Retrieves limit for a specific metric. Retrieves limit for a specific metric.
@ -485,7 +598,8 @@ def get_quota_current_limit(project_link, metric_name):
''' '''
client, interval = create_client() client, interval = create_client()
results = client.list_time_series(request={ results = client.list_time_series(
request={
"name": project_link, "name": project_link,
"filter": f'metric.type = "{metric_name}"', "filter": f'metric.type = "{metric_name}"',
"interval": interval, "interval": interval,
@ -494,6 +608,7 @@ def get_quota_current_limit(project_link, metric_name):
results_list = list(results) results_list = list(results)
return results_list return results_list
def customize_quota_view(quota_results): def customize_quota_view(quota_results):
''' '''
Customize the quota output for an easier parsable output. Customize the quota output for an easier parsable output.
@ -513,6 +628,7 @@ def customize_quota_view(quota_results):
quotaViewList.append(quotaViewJson) quotaViewList.append(quotaViewJson)
return quotaViewList return quotaViewList
def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
''' '''
Updates the network dictionary with quota usage and limit values. Updates the network dictionary with quota usage and limit values.
@ -527,7 +643,8 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
''' '''
if quota_usage: if quota_usage:
for net in quota_usage: for net in quota_usage:
if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same if net['network_id'] == network_dict[
'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
network_dict['usage'] = net['value'] # set network usage in dictionary network_dict['usage'] = net['value'] # set network usage in dictionary
break break
else: else:
@ -537,19 +654,28 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
if quota_limit: if quota_limit:
for net in quota_limit: for net in quota_limit:
if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same if net['network_id'] == network_dict[
'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
network_dict['limit'] = net['value'] # set network limit in dictionary network_dict['limit'] = net['value'] # set network limit in dictionary
break break
else: else:
if network_dict['network name'] in limit_list: # if network limit is in the environmental variables if network_dict[
network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) 'network name'] in limit_list: # if network limit is in the environmental variables
network_dict['limit'] = int(
limit_list[limit_list.index(network_dict['network name']) + 1])
else: else:
network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value network_dict['limit'] = int(
limit_list[limit_list.index('default_value') +
1]) # set default value
else: # if quotas does not appear in GCP quotas else: # if quotas does not appear in GCP quotas
if network_dict['network name'] in limit_list: if network_dict['network name'] in limit_list:
network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) # ["default", 100, "networkname", 200] network_dict['limit'] = int(
limit_list[limit_list.index(network_dict['network name']) +
1]) # ["default", 100, "networkname", 200]
else: else:
network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) network_dict['limit'] = int(limit_list[limit_list.index('default_value') +
1])
def get_usage_limit(network, quota_usage, quota_limit, limit_list): def get_usage_limit(network, quota_usage, quota_limit, limit_list):
''' '''
@ -569,29 +695,36 @@ def get_usage_limit(network, quota_usage, quota_limit, limit_list):
if quota_usage: if quota_usage:
for net in quota_usage: for net in quota_usage:
if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same if net['network_id'] == network[
'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same
usage = net['value'] # set network usage in dictionary usage = net['value'] # set network usage in dictionary
break break
if quota_limit: if quota_limit:
for net in quota_limit: for net in quota_limit:
if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same if net['network_id'] == network[
'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same
limit = net['value'] # set network limit in dictionary limit = net['value'] # set network limit in dictionary
break break
else: else:
if network['network_name'] in limit_list: # if network limit is in the environmental variables if network[
'network_name'] in limit_list: # if network limit is in the environmental variables
limit = int(limit_list[limit_list.index(network['network_name']) + 1]) limit = int(limit_list[limit_list.index(network['network_name']) + 1])
else: else:
limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value limit = int(limit_list[limit_list.index('default_value') +
1]) # set default value
else: # if quotas does not appear in GCP quotas else: # if quotas does not appear in GCP quotas
if network['network_name'] in limit_list: if network['network_name'] in limit_list:
limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200] limit = int(limit_list[limit_list.index(network['network_name']) +
1]) # ["default", 100, "networkname", 200]
else: else:
limit = int(limit_list[limit_list.index('default_value') + 1]) limit = int(limit_list[limit_list.index('default_value') + 1])
return usage, limit return usage, limit
def write_data_to_metric(monitored_project_id, value, metric_name, network_name):
def write_data_to_metric(monitored_project_id, value, metric_name,
network_name):
''' '''
Writes data to Cloud Monitoring custom metrics. Writes data to Cloud Monitoring custom metrics.
@ -615,8 +748,17 @@ def write_data_to_metric(monitored_project_id, value, metric_name, network_name)
now = time.time() now = time.time()
seconds = int(now) seconds = int(now)
nanos = int((now - seconds) * 10**9) nanos = int((now - seconds) * 10**9)
interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}}) interval = monitoring_v3.TimeInterval(
point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) {"end_time": {
"seconds": seconds,
"nanos": nanos
}})
point = monitoring_v3.Point({
"interval": interval,
"value": {
"double_value": value
}
})
series.points = [point] series.points = [point]
client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series])

12
examples/cloud-operations/network-dashboard/main.tf
View File

@ -155,18 +155,6 @@ module "cloud-function" {
} }
} }
# TODO: How to secure Cloud Function invokation? Not member = "allUsers" but specific Scheduler service Account?
# Maybe "service-YOUR_PROJECT_NUMBER@gcp-sa-cloudscheduler.iam.gserviceaccount.com"?
resource "google_cloudfunctions_function_iam_member" "invoker" {
project = module.project-monitoring.project_id
region = var.region
cloud_function = module.cloud-function.function_name
role = "roles/cloudfunctions.invoker"
member = "allUsers"
}
################################################ ################################################
# Cloud Monitoring Dashboard creation # # Cloud Monitoring Dashboard creation #
################################################ ################################################