refactoring main.py
This commit is contained in:
Aurélien Legrand
Julio Castillo
parent
7802c33988
commit
7874f9c02b
|
|
@ -14,15 +14,12 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import os
|
||||
import yaml
|
||||
from google.cloud import monitoring_v3
|
||||
from googleapiclient import discovery
|
||||
from google.api import metric_pb2 as ga_metric
|
||||
import time
|
||||
import os
|
||||
import google.api_core
|
||||
import re
|
||||
import random
|
||||
import yaml
|
||||
|
||||
# list of projects from which function will get quotas information
|
||||
MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",")
|
||||
|
|
@ -41,6 +38,17 @@ LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",")
|
|||
LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",")
|
||||
LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",")
|
||||
|
||||
# Existing GCP metrics per network
|
||||
L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
|
||||
L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
|
||||
L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage"
|
||||
L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit"
|
||||
SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage"
|
||||
SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
|
||||
GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
|
||||
GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
|
||||
|
||||
|
||||
def main(event, context):
|
||||
'''
|
||||
Cloud Function Entry point, called by the scheduler.
|
||||
|
|
@ -59,32 +67,31 @@ def main(event, context):
|
|||
get_vpc_peering_data(metrics_dict)
|
||||
get_l4_forwarding_rules_data(metrics_dict)
|
||||
|
||||
# Existing GCP Monitoring metrics for L4 Forwarding Rules per Network
|
||||
l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
|
||||
l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
|
||||
get_pgg_data(
|
||||
metrics_dict["metrics_per_peering_group"]
|
||||
["l4_forwarding_rules_per_peering_group"],
|
||||
L4_FORWARDING_RULES_USAGE_METRIC, L4_FORWARDING_RULES_LIMIT_METRIC,
|
||||
LIMIT_L4_PPG)
|
||||
|
||||
get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, LIMIT_L4_PPG)
|
||||
get_pgg_data(
|
||||
metrics_dict["metrics_per_peering_group"]
|
||||
["l7_forwarding_rules_per_peering_group"],
|
||||
L7_FORWARDING_RULES_USAGE_METRIC, L7_FORWARDING_RULES_LIMIT_METRIC,
|
||||
LIMIT_L7_PPG)
|
||||
|
||||
# Existing GCP Monitoring metrics for L7 Forwarding Rules per Network
|
||||
l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage"
|
||||
l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit"
|
||||
get_pgg_data(
|
||||
metrics_dict["metrics_per_peering_group"]
|
||||
["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC,
|
||||
SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS)
|
||||
|
||||
get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, LIMIT_L7_PPG)
|
||||
|
||||
# Existing GCP Monitoring metrics for Subnet Ranges per Network
|
||||
subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage"
|
||||
subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
|
||||
|
||||
get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, LIMIT_SUBNETS)
|
||||
|
||||
# Existing GCP Monitoring metrics for GCE per Network
|
||||
gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
|
||||
gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
|
||||
|
||||
get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, LIMIT_INSTANCES_PPG)
|
||||
get_pgg_data(
|
||||
metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"],
|
||||
GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC,
|
||||
LIMIT_INSTANCES_PPG)
|
||||
|
||||
return 'Function executed successfully'
|
||||
|
||||
|
||||
def create_client():
|
||||
'''
|
||||
Creates the monitoring API client, that will be used to create, read and update custom metrics.
|
||||
|
|
@ -99,16 +106,22 @@ def create_client():
|
|||
client = monitoring_v3.MetricServiceClient()
|
||||
now = time.time()
|
||||
seconds = int(now)
|
||||
nanos = int((now - seconds) * 10 ** 9)
|
||||
interval = monitoring_v3.TimeInterval(
|
||||
{
|
||||
"end_time": {"seconds": seconds, "nanos": nanos},
|
||||
"start_time": {"seconds": (seconds - 86400), "nanos": nanos},
|
||||
nanos = int((now - seconds) * 10**9)
|
||||
interval = monitoring_v3.TimeInterval({
|
||||
"end_time": {
|
||||
"seconds": seconds,
|
||||
"nanos": nanos
|
||||
},
|
||||
"start_time": {
|
||||
"seconds": (seconds - 86400),
|
||||
"nanos": nanos
|
||||
},
|
||||
})
|
||||
return (client, interval)
|
||||
except Exception as e:
|
||||
raise Exception("Error occurred creating the client: {}".format(e))
|
||||
|
||||
|
||||
def create_metrics():
|
||||
with open("metrics.yaml", 'r') as stream:
|
||||
try:
|
||||
|
|
@ -123,6 +136,7 @@ def create_metrics():
|
|||
except yaml.YAMLError as exc:
|
||||
print(exc)
|
||||
|
||||
|
||||
def create_metric(metric_name, description):
|
||||
'''
|
||||
Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
|
||||
|
|
@ -147,9 +161,11 @@ def create_metric(metric_name, description):
|
|||
descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE
|
||||
descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE
|
||||
descriptor.description = description
|
||||
descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, metric_descriptor=descriptor)
|
||||
descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK,
|
||||
metric_descriptor=descriptor)
|
||||
print("Created {}.".format(descriptor.name))
|
||||
|
||||
|
||||
def get_gce_instances_data(metrics_dict):
|
||||
'''
|
||||
Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric.
|
||||
|
|
@ -166,17 +182,27 @@ def get_gce_instances_data(metrics_dict):
|
|||
for project in MONITORED_PROJECTS_LIST:
|
||||
network_dict = get_networks(project)
|
||||
|
||||
current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage)
|
||||
current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit)
|
||||
current_quota_usage = get_quota_current_usage(f"projects/{project}",
|
||||
metric_instances_usage)
|
||||
current_quota_limit = get_quota_current_limit(f"projects/{project}",
|
||||
metric_instances_limit)
|
||||
|
||||
current_quota_usage_view = customize_quota_view(current_quota_usage)
|
||||
current_quota_limit_view = customize_quota_view(current_quota_limit)
|
||||
|
||||
for net in network_dict:
|
||||
set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_INSTANCES)
|
||||
write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["instance_per_network"]["usage"]["name"], net['network name'])
|
||||
write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["limit"]["name"], net['network name'])
|
||||
write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["utilization"]["name"], net['network name'])
|
||||
set_usage_limits(net, current_quota_usage_view, current_quota_limit_view,
|
||||
LIMIT_INSTANCES)
|
||||
write_data_to_metric(
|
||||
project, net['usage'], metrics_dict["metrics_per_network"]
|
||||
["instance_per_network"]["usage"]["name"], net['network name'])
|
||||
write_data_to_metric(
|
||||
project, net['limit'], metrics_dict["metrics_per_network"]
|
||||
["instance_per_network"]["limit"]["name"], net['network name'])
|
||||
write_data_to_metric(
|
||||
project, net['usage'] / net['limit'],
|
||||
metrics_dict["metrics_per_network"]["instance_per_network"]
|
||||
["utilization"]["name"], net['network name'])
|
||||
|
||||
print(f"Wrote number of instances to metric for projects/{project}")
|
||||
|
||||
|
|
@ -191,19 +217,38 @@ def get_vpc_peering_data(metrics_dict):
|
|||
None
|
||||
'''
|
||||
for project in MONITORED_PROJECTS_LIST:
|
||||
active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, LIMIT_VPC_PEER)
|
||||
active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(
|
||||
project, LIMIT_VPC_PEER)
|
||||
for peering in active_vpc_peerings:
|
||||
write_data_to_metric(project, peering['active_peerings'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["usage"]["name"], peering['network_name'])
|
||||
write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["limit"]["name"], peering['network_name'])
|
||||
write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["utilization"]["name"], peering['network_name'])
|
||||
print("Wrote number of active VPC peerings to custom metric for project:", project)
|
||||
write_data_to_metric(
|
||||
project, peering['active_peerings'],
|
||||
metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
|
||||
["usage"]["name"], peering['network_name'])
|
||||
write_data_to_metric(
|
||||
project, peering['network_limit'], metrics_dict["metrics_per_network"]
|
||||
["vpc_peering_active_per_network"]["limit"]["name"],
|
||||
peering['network_name'])
|
||||
write_data_to_metric(
|
||||
project, peering['active_peerings'] / peering['network_limit'],
|
||||
metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]
|
||||
["utilization"]["name"], peering['network_name'])
|
||||
print("Wrote number of active VPC peerings to custom metric for project:",
|
||||
project)
|
||||
|
||||
for peering in vpc_peerings:
|
||||
write_data_to_metric(project, peering['peerings'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["usage"]["name"], peering['network_name'])
|
||||
write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["limit"]["name"], peering['network_name'])
|
||||
write_data_to_metric(project, peering['peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["utilization"]["name"], peering['network_name'])
|
||||
write_data_to_metric(
|
||||
project, peering['peerings'], metrics_dict["metrics_per_network"]
|
||||
["vpc_peering_per_network"]["usage"]["name"], peering['network_name'])
|
||||
write_data_to_metric(
|
||||
project, peering['network_limit'], metrics_dict["metrics_per_network"]
|
||||
["vpc_peering_per_network"]["limit"]["name"], peering['network_name'])
|
||||
write_data_to_metric(
|
||||
project, peering['peerings'] / peering['network_limit'],
|
||||
metrics_dict["metrics_per_network"]["vpc_peering_per_network"]
|
||||
["utilization"]["name"], peering['network_name'])
|
||||
print("Wrote number of VPC peerings to custom metric for project:", project)
|
||||
|
||||
|
||||
def gather_vpc_peerings_data(project_id, limit_list):
|
||||
'''
|
||||
Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
|
||||
|
|
@ -233,13 +278,24 @@ def gather_vpc_peerings_data(project_id, limit_list):
|
|||
peerings_count = 0
|
||||
active_peerings_count = 0
|
||||
|
||||
active_d = {'project_id': project_id,'network_name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)}
|
||||
active_d = {
|
||||
'project_id': project_id,
|
||||
'network_name': network['name'],
|
||||
'active_peerings': active_peerings_count,
|
||||
'network_limit': get_limit(network['name'], limit_list)
|
||||
}
|
||||
active_peerings_dict.append(active_d)
|
||||
d = {'project_id': project_id,'network_name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)}
|
||||
d = {
|
||||
'project_id': project_id,
|
||||
'network_name': network['name'],
|
||||
'peerings': peerings_count,
|
||||
'network_limit': get_limit(network['name'], limit_list)
|
||||
}
|
||||
peerings_dict.append(d)
|
||||
|
||||
return active_peerings_dict, peerings_dict
|
||||
|
||||
|
||||
def get_limit(network_name, limit_list):
|
||||
'''
|
||||
Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
|
||||
|
|
@ -258,6 +314,7 @@ def get_limit(network_name, limit_list):
|
|||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def get_l4_forwarding_rules_data(metrics_dict):
|
||||
'''
|
||||
Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
|
||||
|
|
@ -274,19 +331,33 @@ def get_l4_forwarding_rules_data(metrics_dict):
|
|||
for project in MONITORED_PROJECTS_LIST:
|
||||
network_dict = get_networks(project)
|
||||
|
||||
current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage)
|
||||
current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit)
|
||||
current_quota_usage = get_quota_current_usage(f"projects/{project}",
|
||||
l4_forwarding_rules_usage)
|
||||
current_quota_limit = get_quota_current_limit(f"projects/{project}",
|
||||
l4_forwarding_rules_limit)
|
||||
|
||||
current_quota_usage_view = customize_quota_view(current_quota_usage)
|
||||
current_quota_limit_view = customize_quota_view(current_quota_limit)
|
||||
|
||||
for net in network_dict:
|
||||
set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_L4)
|
||||
write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name'])
|
||||
write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name'])
|
||||
write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["utilization"]["name"], net['network name'])
|
||||
set_usage_limits(net, current_quota_usage_view, current_quota_limit_view,
|
||||
LIMIT_L4)
|
||||
write_data_to_metric(
|
||||
project, net['usage'], metrics_dict["metrics_per_network"]
|
||||
["l4_forwarding_rules_per_network"]["usage"]["name"],
|
||||
net['network name'])
|
||||
write_data_to_metric(
|
||||
project, net['limit'], metrics_dict["metrics_per_network"]
|
||||
["l4_forwarding_rules_per_network"]["limit"]["name"],
|
||||
net['network name'])
|
||||
write_data_to_metric(
|
||||
project, net['usage'] / net['limit'],
|
||||
metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]
|
||||
["utilization"]["name"], net['network name'])
|
||||
|
||||
print(
|
||||
f"Wrote number of L4 forwarding rules to metric for projects/{project}")
|
||||
|
||||
print(f"Wrote number of L4 forwarding rules to metric for projects/{project}")
|
||||
|
||||
def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
|
||||
'''
|
||||
|
|
@ -310,31 +381,46 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
|
|||
|
||||
# For each network in this GCP project
|
||||
for network_dict in network_dict_list:
|
||||
current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric)
|
||||
current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric)
|
||||
current_quota_usage = get_quota_current_usage(f"projects/{project}",
|
||||
usage_metric)
|
||||
current_quota_limit = get_quota_current_limit(f"projects/{project}",
|
||||
limit_metric)
|
||||
|
||||
current_quota_usage_view = customize_quota_view(current_quota_usage)
|
||||
current_quota_limit_view = customize_quota_view(current_quota_limit)
|
||||
|
||||
usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg)
|
||||
usage, limit = get_usage_limit(network_dict, current_quota_usage_view,
|
||||
current_quota_limit_view, limit_ppg)
|
||||
# Here we add usage and limit to the network dictionary
|
||||
network_dict["usage"] = usage
|
||||
network_dict["limit"] = limit
|
||||
|
||||
# For every peered network, get usage and limits
|
||||
for peered_network in network_dict['peerings']:
|
||||
peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric))
|
||||
peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric))
|
||||
peering_project_usage = customize_quota_view(
|
||||
get_quota_current_usage(f"projects/{peered_network['project_id']}",
|
||||
usage_metric))
|
||||
peering_project_limit = customize_quota_view(
|
||||
get_quota_current_limit(f"projects/{peered_network['project_id']}",
|
||||
limit_metric))
|
||||
|
||||
usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg)
|
||||
usage, limit = get_usage_limit(peered_network, peering_project_usage,
|
||||
peering_project_limit, limit_ppg)
|
||||
# Here we add usage and limit to the peered network dictionary
|
||||
peered_network["usage"] = usage
|
||||
peered_network["limit"] = limit
|
||||
|
||||
count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_ppg)
|
||||
print(f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}")
|
||||
count_effective_limit(project, network_dict, metric_dict["usage"]["name"],
|
||||
metric_dict["limit"]["name"],
|
||||
metric_dict["utilization"]["name"], limit_ppg)
|
||||
print(
|
||||
f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}"
|
||||
)
|
||||
|
||||
def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg):
|
||||
|
||||
def count_effective_limit(project_id, network_dict, usage_metric_name,
|
||||
limit_metric_name, utilization_metric_name,
|
||||
limit_ppg):
|
||||
'''
|
||||
Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
|
||||
Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
|
||||
|
|
@ -359,12 +445,15 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met
|
|||
peering_group_usage += peered_network['usage']
|
||||
|
||||
# Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit)
|
||||
limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg))
|
||||
limit_step1 = max(network_dict['limit'],
|
||||
get_limit(network_dict['network_name'], limit_ppg))
|
||||
|
||||
# Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network
|
||||
limit_step2 = []
|
||||
for peered_network in network_dict['peerings']:
|
||||
limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg)))
|
||||
limit_step2.append(
|
||||
max(peered_network['limit'],
|
||||
get_limit(peered_network['network_name'], limit_ppg)))
|
||||
|
||||
# Calculates effective limit: Step 3: Find minimum from the list created by Step 2
|
||||
limit_step3 = min(limit_step2)
|
||||
|
|
@ -373,9 +462,13 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met
|
|||
effective_limit = max(limit_step1, limit_step3)
|
||||
utilization = peering_group_usage / effective_limit
|
||||
|
||||
write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name'])
|
||||
write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name'])
|
||||
write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name'])
|
||||
write_data_to_metric(project_id, peering_group_usage, usage_metric_name,
|
||||
network_dict['network_name'])
|
||||
write_data_to_metric(project_id, effective_limit, limit_metric_name,
|
||||
network_dict['network_name'])
|
||||
write_data_to_metric(project_id, utilization, utilization_metric_name,
|
||||
network_dict['network_name'])
|
||||
|
||||
|
||||
def get_networks(project_id):
|
||||
'''
|
||||
|
|
@ -393,10 +486,11 @@ def get_networks(project_id):
|
|||
for network in response['items']:
|
||||
NETWORK = network['name']
|
||||
ID = network['id']
|
||||
d = {'project_id':project_id,'network name':NETWORK,'network id':ID}
|
||||
d = {'project_id': project_id, 'network name': NETWORK, 'network id': ID}
|
||||
network_dict.append(d)
|
||||
return network_dict
|
||||
|
||||
|
||||
def gather_peering_data(project_id):
|
||||
'''
|
||||
Returns a dictionary of all peerings for all networks in a project.
|
||||
|
|
@ -412,20 +506,36 @@ def gather_peering_data(project_id):
|
|||
network_list = []
|
||||
if 'items' in response:
|
||||
for network in response['items']:
|
||||
net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]}
|
||||
net = {
|
||||
'project_id': project_id,
|
||||
'network_name': network['name'],
|
||||
'network_id': network['id'],
|
||||
'peerings': []
|
||||
}
|
||||
if 'peerings' in network:
|
||||
STATE = network['peerings'][0]['state']
|
||||
if STATE == "ACTIVE":
|
||||
for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}"
|
||||
start = peered_network['network'].find("projects/") + len('projects/')
|
||||
for peered_network in network[
|
||||
'peerings']: # "projects/{project_name}/global/networks/{network_name}"
|
||||
start = peered_network['network'].find("projects/") + len(
|
||||
'projects/')
|
||||
end = peered_network['network'].find("/global")
|
||||
peered_project = peered_network['network'][start:end]
|
||||
peered_network_name = peered_network['network'].split("networks/")[1]
|
||||
peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)}
|
||||
peered_network_name = peered_network['network'].split(
|
||||
"networks/")[1]
|
||||
peered_net = {
|
||||
'project_id':
|
||||
peered_project,
|
||||
'network_name':
|
||||
peered_network_name,
|
||||
'network_id':
|
||||
get_network_id(peered_project, peered_network_name)
|
||||
}
|
||||
net["peerings"].append(peered_net)
|
||||
network_list.append(net)
|
||||
return network_list
|
||||
|
||||
|
||||
def get_network_id(project_id, network_name):
|
||||
'''
|
||||
Returns the network_id for a specific project / network name.
|
||||
|
|
@ -452,6 +562,7 @@ def get_network_id(project_id, network_name):
|
|||
|
||||
return network_id
|
||||
|
||||
|
||||
def get_quota_current_usage(project_link, metric_name):
|
||||
'''
|
||||
Retrieves quota usage for a specific metric.
|
||||
|
|
@ -464,7 +575,8 @@ def get_quota_current_usage(project_link, metric_name):
|
|||
'''
|
||||
client, interval = create_client()
|
||||
|
||||
results = client.list_time_series(request={
|
||||
results = client.list_time_series(
|
||||
request={
|
||||
"name": project_link,
|
||||
"filter": f'metric.type = "{metric_name}"',
|
||||
"interval": interval,
|
||||
|
|
@ -473,6 +585,7 @@ def get_quota_current_usage(project_link, metric_name):
|
|||
results_list = list(results)
|
||||
return (results_list)
|
||||
|
||||
|
||||
def get_quota_current_limit(project_link, metric_name):
|
||||
'''
|
||||
Retrieves limit for a specific metric.
|
||||
|
|
@ -485,7 +598,8 @@ def get_quota_current_limit(project_link, metric_name):
|
|||
'''
|
||||
client, interval = create_client()
|
||||
|
||||
results = client.list_time_series(request={
|
||||
results = client.list_time_series(
|
||||
request={
|
||||
"name": project_link,
|
||||
"filter": f'metric.type = "{metric_name}"',
|
||||
"interval": interval,
|
||||
|
|
@ -494,6 +608,7 @@ def get_quota_current_limit(project_link, metric_name):
|
|||
results_list = list(results)
|
||||
return results_list
|
||||
|
||||
|
||||
def customize_quota_view(quota_results):
|
||||
'''
|
||||
Customize the quota output for an easier parsable output.
|
||||
|
|
@ -513,6 +628,7 @@ def customize_quota_view(quota_results):
|
|||
quotaViewList.append(quotaViewJson)
|
||||
return quotaViewList
|
||||
|
||||
|
||||
def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
|
||||
'''
|
||||
Updates the network dictionary with quota usage and limit values.
|
||||
|
|
@ -527,7 +643,8 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
|
|||
'''
|
||||
if quota_usage:
|
||||
for net in quota_usage:
|
||||
if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
if net['network_id'] == network_dict[
|
||||
'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
network_dict['usage'] = net['value'] # set network usage in dictionary
|
||||
break
|
||||
else:
|
||||
|
|
@ -537,19 +654,28 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
|
|||
|
||||
if quota_limit:
|
||||
for net in quota_limit:
|
||||
if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
if net['network_id'] == network_dict[
|
||||
'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
network_dict['limit'] = net['value'] # set network limit in dictionary
|
||||
break
|
||||
else:
|
||||
if network_dict['network name'] in limit_list: # if network limit is in the environmental variables
|
||||
network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1])
|
||||
if network_dict[
|
||||
'network name'] in limit_list: # if network limit is in the environmental variables
|
||||
network_dict['limit'] = int(
|
||||
limit_list[limit_list.index(network_dict['network name']) + 1])
|
||||
else:
|
||||
network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value
|
||||
network_dict['limit'] = int(
|
||||
limit_list[limit_list.index('default_value') +
|
||||
1]) # set default value
|
||||
else: # if quotas does not appear in GCP quotas
|
||||
if network_dict['network name'] in limit_list:
|
||||
network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) # ["default", 100, "networkname", 200]
|
||||
network_dict['limit'] = int(
|
||||
limit_list[limit_list.index(network_dict['network name']) +
|
||||
1]) # ["default", 100, "networkname", 200]
|
||||
else:
|
||||
network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1])
|
||||
network_dict['limit'] = int(limit_list[limit_list.index('default_value') +
|
||||
1])
|
||||
|
||||
|
||||
def get_usage_limit(network, quota_usage, quota_limit, limit_list):
|
||||
'''
|
||||
|
|
@ -569,29 +695,36 @@ def get_usage_limit(network, quota_usage, quota_limit, limit_list):
|
|||
|
||||
if quota_usage:
|
||||
for net in quota_usage:
|
||||
if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
if net['network_id'] == network[
|
||||
'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
usage = net['value'] # set network usage in dictionary
|
||||
break
|
||||
|
||||
if quota_limit:
|
||||
for net in quota_limit:
|
||||
if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
if net['network_id'] == network[
|
||||
'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same
|
||||
limit = net['value'] # set network limit in dictionary
|
||||
break
|
||||
else:
|
||||
if network['network_name'] in limit_list: # if network limit is in the environmental variables
|
||||
if network[
|
||||
'network_name'] in limit_list: # if network limit is in the environmental variables
|
||||
limit = int(limit_list[limit_list.index(network['network_name']) + 1])
|
||||
else:
|
||||
limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value
|
||||
limit = int(limit_list[limit_list.index('default_value') +
|
||||
1]) # set default value
|
||||
else: # if quotas does not appear in GCP quotas
|
||||
if network['network_name'] in limit_list:
|
||||
limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200]
|
||||
limit = int(limit_list[limit_list.index(network['network_name']) +
|
||||
1]) # ["default", 100, "networkname", 200]
|
||||
else:
|
||||
limit = int(limit_list[limit_list.index('default_value') + 1])
|
||||
|
||||
return usage, limit
|
||||
|
||||
def write_data_to_metric(monitored_project_id, value, metric_name, network_name):
|
||||
|
||||
def write_data_to_metric(monitored_project_id, value, metric_name,
|
||||
network_name):
|
||||
'''
|
||||
Writes data to Cloud Monitoring custom metrics.
|
||||
|
||||
|
|
@ -614,9 +747,18 @@ def write_data_to_metric(monitored_project_id, value, metric_name, network_name)
|
|||
|
||||
now = time.time()
|
||||
seconds = int(now)
|
||||
nanos = int((now - seconds) * 10 ** 9)
|
||||
interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}})
|
||||
point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}})
|
||||
nanos = int((now - seconds) * 10**9)
|
||||
interval = monitoring_v3.TimeInterval(
|
||||
{"end_time": {
|
||||
"seconds": seconds,
|
||||
"nanos": nanos
|
||||
}})
|
||||
point = monitoring_v3.Point({
|
||||
"interval": interval,
|
||||
"value": {
|
||||
"double_value": value
|
||||
}
|
||||
})
|
||||
series.points = [point]
|
||||
|
||||
client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series])
|
||||
|
|
@ -155,18 +155,6 @@ module "cloud-function" {
|
|||
}
|
||||
}
|
||||
|
||||
# TODO: How to secure Cloud Function invokation? Not member = "allUsers" but specific Scheduler service Account?
|
||||
# Maybe "service-YOUR_PROJECT_NUMBER@gcp-sa-cloudscheduler.iam.gserviceaccount.com"?
|
||||
|
||||
resource "google_cloudfunctions_function_iam_member" "invoker" {
|
||||
project = module.project-monitoring.project_id
|
||||
region = var.region
|
||||
cloud_function = module.cloud-function.function_name
|
||||
|
||||
role = "roles/cloudfunctions.invoker"
|
||||
member = "allUsers"
|
||||
}
|
||||
|
||||
################################################
|
||||
# Cloud Monitoring Dashboard creation #
|
||||
################################################
|
||||
|
|
|
|||
Loading…
Reference in New Issue