237 lines
9.2 KiB
Python
237 lines
9.2 KiB
Python
#
|
|
# Copyright 2022 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
import time
|
|
|
|
from google.api_core import exceptions
|
|
from google.cloud import monitoring_v3
|
|
from . import metrics
|
|
|
|
|
|
def get_quotas_dict(quotas_list):
|
|
'''
|
|
Creates a dictionary of quotas from a list, with lower case quota name as keys
|
|
Parameters:
|
|
quotas_array (array): array of quotas
|
|
Returns:
|
|
quotas_dict (dict): dictionary of quotas
|
|
'''
|
|
quota_keys = [q['metric'] for q in quotas_list]
|
|
quotas_dict = dict()
|
|
i = 0
|
|
for key in quota_keys:
|
|
if ("metric" in quotas_list[i]):
|
|
del (quotas_list[i]["metric"])
|
|
quotas_dict[key.lower()] = quotas_list[i]
|
|
i += 1
|
|
return quotas_dict
|
|
|
|
|
|
def get_quota_project_limit(config, regions=["global"]):
|
|
'''
|
|
Retrieves quotas for all monitored project in selected regions, default 'global'
|
|
Parameters:
|
|
project_link (string): Project link.
|
|
Returns:
|
|
quotas (dict): quotas for all selected regions, default 'global'
|
|
'''
|
|
try:
|
|
request = {}
|
|
quotas = dict()
|
|
for project in config["monitored_projects"]:
|
|
quotas[project] = dict()
|
|
if regions != ["global"]:
|
|
for region in regions:
|
|
request = config["clients"]["discovery_client"].compute.regions().get(
|
|
region=region, project=project)
|
|
response = request.execute()
|
|
quotas[project][region] = get_quotas_dict(response['quotas'])
|
|
else:
|
|
region = "global"
|
|
request = config["clients"]["discovery_client"].projects().get(
|
|
project=project, fields="quotas")
|
|
response = request.execute()
|
|
quotas[project][region] = get_quotas_dict(response['quotas'])
|
|
|
|
return quotas
|
|
except exceptions.PermissionDenied as err:
|
|
print(
|
|
f"Warning: error reading quotas for {project}. " +
|
|
f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
|
|
)
|
|
return None
|
|
|
|
|
|
def get_ppg(network_link, limit_dict):
|
|
'''
|
|
Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
|
|
|
|
Parameters:
|
|
network_link (string): VPC network link.
|
|
limit_list (list of string): Used to get the limit per VPC or the default limit.
|
|
Returns:
|
|
limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
|
|
'''
|
|
if network_link in limit_dict:
|
|
return limit_dict[network_link]
|
|
else:
|
|
if 'default_value' in limit_dict:
|
|
return limit_dict['default_value']
|
|
else:
|
|
print(f"Error: limit not found for {network_link}")
|
|
return 0
|
|
|
|
|
|
def set_limits(network_dict, quota_limit, limit_dict):
|
|
'''
|
|
Updates the network dictionary with quota limit values.
|
|
|
|
Parameters:
|
|
network_dict (dictionary of string: string): Contains network information.
|
|
quota_limit (list of dictionaries of string: string): Current quota limit.
|
|
limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
|
|
Returns:
|
|
None
|
|
'''
|
|
|
|
network_dict['limit'] = None
|
|
|
|
if quota_limit:
|
|
for net in quota_limit:
|
|
if net['network_id'] == network_dict['network_id']:
|
|
network_dict['limit'] = net['value']
|
|
return
|
|
|
|
network_link = f"https://www.googleapis.com/compute/v1/projects/{network_dict['project_id']}/global/networks/{network_dict['network_name']}"
|
|
|
|
if network_link in limit_dict:
|
|
network_dict['limit'] = limit_dict[network_link]
|
|
else:
|
|
if 'default_value' in limit_dict:
|
|
network_dict['limit'] = limit_dict['default_value']
|
|
else:
|
|
print(f"Error: Couldn't find limit for {network_link}")
|
|
network_dict['limit'] = 0
|
|
|
|
|
|
def get_quota_current_limit(config, project_link, metric_name):
|
|
'''
|
|
Retrieves limit for a specific metric.
|
|
|
|
Parameters:
|
|
project_link (string): Project link.
|
|
metric_name (string): Name of the metric.
|
|
Returns:
|
|
results_list (list of string): Current limit.
|
|
'''
|
|
|
|
try:
|
|
results = config["clients"]["monitoring_client"].list_time_series(
|
|
request={
|
|
"name": project_link,
|
|
"filter": f'metric.type = "{metric_name}"',
|
|
"interval": config["monitoring_interval"],
|
|
"view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL
|
|
})
|
|
results_list = list(results)
|
|
return results_list
|
|
except exceptions.PermissionDenied as err:
|
|
print(
|
|
f"Warning: error reading quotas for {project_link}. " +
|
|
f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
|
|
)
|
|
return None
|
|
|
|
|
|
def count_effective_limit(config, project_id, network_dict, usage_metric_name,
|
|
limit_metric_name, utilization_metric_name,
|
|
limit_dict, timestamp=None):
|
|
'''
|
|
Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
|
|
Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
|
|
|
|
Parameters:
|
|
config (dict): The dict containing config like clients and limits
|
|
project_id (string): Project ID for the project to be analyzed.
|
|
network_dict (dictionary of string: string): Contains all required information about the network to get the usage, limit and utilization.
|
|
usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group.
|
|
limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group.
|
|
utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group.
|
|
limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit).
|
|
timestamp (time): timestamp to be recorded for all points
|
|
Returns:
|
|
None
|
|
'''
|
|
|
|
if timestamp == None:
|
|
timestamp = time.time()
|
|
|
|
if network_dict['peerings'] == []:
|
|
return
|
|
|
|
# Get usage: Sums usage for current network + all peered networks
|
|
peering_group_usage = network_dict['usage']
|
|
for peered_network in network_dict['peerings']:
|
|
if 'usage' not in peered_network:
|
|
print(
|
|
f"Can not add metrics for peered network in projects/{project_id} as no usage metrics exist due to missing permissions"
|
|
)
|
|
continue
|
|
peering_group_usage += peered_network['usage']
|
|
|
|
network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}"
|
|
|
|
# Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit)
|
|
limit_step1 = max(network_dict['limit'], get_ppg(network_link, limit_dict))
|
|
|
|
# Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network
|
|
limit_step2 = []
|
|
for peered_network in network_dict['peerings']:
|
|
peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}"
|
|
|
|
if 'limit' in peered_network:
|
|
limit_step2.append(
|
|
max(peered_network['limit'], get_ppg(peered_network_link,
|
|
limit_dict)))
|
|
else:
|
|
print(
|
|
f"Ignoring projects/{peered_network['project_id']} for limits in peering group of project {project_id} as no limits are available."
|
|
+
|
|
"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project"
|
|
)
|
|
|
|
# Calculates effective limit: Step 3: Find minimum from the list created by Step 2
|
|
limit_step3 = 0
|
|
if len(limit_step2) > 0:
|
|
limit_step3 = min(limit_step2)
|
|
|
|
# Calculates effective limit: Step 4: Find maximum from step 1 and step 3
|
|
effective_limit = max(limit_step1, limit_step3)
|
|
utilization = peering_group_usage / effective_limit
|
|
metric_labels = {
|
|
'project': project_id,
|
|
'network_name': network_dict['network_name']
|
|
}
|
|
metrics.append_data_to_series_buffer(config, usage_metric_name,
|
|
peering_group_usage, metric_labels,
|
|
timestamp=timestamp)
|
|
metrics.append_data_to_series_buffer(config, limit_metric_name,
|
|
effective_limit, metric_labels,
|
|
timestamp=timestamp)
|
|
metrics.append_data_to_series_buffer(config, utilization_metric_name,
|
|
utilization, metric_labels,
|
|
timestamp=timestamp)
|