zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added support for ppg static routes

This commit is contained in:
Maurizio Noseda Pedraglio 2022-10-12 14:51:03 +02:00
parent 311d50c183
commit 61917690e5
5 changed files with 115 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
blueprints/cloud-operations/network-dashboard/README.md
View File

@ -42,21 +42,24 @@ The Cloud Function currently tracks usage, limit and utilization of:
- internal forwarding rules for internal L7 load balancers per VPC
- internal forwarding rules for internal L4 load balancers per VPC peering group
- internal forwarding rules for internal L7 load balancers per VPC peering group
- Dynamic routes per VPC
- Dynamic routes per VPC peering group
- Dynamic routes per VPC (note: assumes global routing is ON)
- Dynamic routes per VPC peering group (note: assumes custom routes importing/exporting is ON)
- Static routes per project (VPC drill down is available for usage)
- Static routes per VPC peering group (note: assumes custom routes sharing is ON for all peered networks)
- IP utilization per subnet (% of IP addresses used in a subnet)
- VPC firewall rules per project (VPC drill down is available for usage)
- Tuples per Firewall Policy
It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring.
Note that metrics are created in the cloud-function/metrics.yaml file.
Note that metrics are created in the cloud-function/metrics.yaml file. also note that the Cloud Function assumes all VPCs in peering groups are within the same organization.
You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`.
## Next steps and ideas
In a future release, we could support:
- Static routes per VPC / per VPC peering group
- Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore)
- Dynamic routes calculation for VPCs/PPGs with "global routing" set to OFF
- Static routes calculation for projects/PPGs with "custom routes importing/exporting" set to OFF
If you are interested in this and/or would like to contribute, please contact legranda@google.com.

16
blueprints/cloud-operations/network-dashboard/cloud-function/main.py
View File

@ -158,6 +158,9 @@ def main(event, context):
l4_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L4")
l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7")
subnet_range_dict = networks.get_subnet_ranges_dict(config)
static_routes_dict = routes.get_static_routes_dict(config)
dynamic_routes_dict = routes.get_dynamic_routes(
config, metrics_dict, limits_dict['dynamic_routes_per_network_limit'])
try:
@ -177,12 +180,11 @@ def main(event, context):
config, metrics_dict, l7_forwarding_rules_dict,
limits_dict['internal_forwarding_rules_l7_limit'], "L7")
routes.get_static_routes_vpc(config, metrics_dict, project_quotas_dict)
routes.get_static_routes_data(config, metrics_dict, static_routes_dict,
project_quotas_dict)
peerings.get_vpc_peering_data(config, metrics_dict,
limits_dict['number_of_vpc_peerings_limit'])
dynamic_routes_dict = routes.get_dynamic_routes(
config, metrics_dict, limits_dict['dynamic_routes_per_network_limit'])
# Per VPC peering group metrics
metrics.get_pgg_data(
@ -205,7 +207,13 @@ def main(event, context):
["subnet_ranges_per_peering_group"], subnet_range_dict,
config["limit_names"]["SUBNET_RANGES"],
limits_dict['number_of_subnet_IP_ranges_ppg_limit'])
routes.get_dynamic_routes_ppg(
#static
routes.get_routes_ppg(
config, metrics_dict["metrics_per_peering_group"]
["static_routes_per_peering_group"], static_routes_dict,
limits_dict['static_routes_per_peering_group_limit'])
#dynamic
routes.get_routes_ppg(
config, metrics_dict["metrics_per_peering_group"]
["dynamic_routes_per_peering_group"], dynamic_routes_dict,
limits_dict['dynamic_routes_per_peering_group_limit'])

12
blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml
View File

@ -172,6 +172,18 @@ metrics_per_peering_group:
utilization:
name: dynamic_routes_per_peering_group_utilization
description: Number of Dynamic routes per peering group - utilization.
static_routes_per_peering_group:
usage:
name: static_routes_per_peering_group_usage
description: Number of Static routes per peering group - usage.
limit:
name: static_routes_per_peering_group_limit
description: Number of Static routes per peering group - limit.
values:
default_value: 250
utilization:
name: static_routes_per_peering_group_utilization
description: Number of Static routes per peering group - utilization.
metrics_per_project:
firewalls:
usage:

11
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py
View File

@ -28,8 +28,8 @@ from . import metrics, networks, limits
def get_firewall_policies_dict(config: dict):
'''
Calls the Asset Inventory API to get all Firewall Policies under the GCP organization
Calls the Asset Inventory API to get all Firewall Policies under the GCP organization, including children
Ignores monitored projects list: returns all policies regardless of their parent resource
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
@ -57,8 +57,8 @@ def get_firewall_policies_dict(config: dict):
def get_firewal_policies_data(config, metrics_dict, firewall_policies_dict):
'''
Gets the data for VPC Firewall lorem ipsum
Gets the data for VPC Firewall Policies in an organization, including children. All folders are considered,
only projects in the monitored projects list are considered.
Parameters:
config (dict): The dict containing config like clients and limits
metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
@ -93,6 +93,9 @@ def get_firewal_policies_data(config, metrics_dict, firewall_policies_dict):
parent_type = re.search("(^\w+)", firewall_policy["parent"]).group(
1) if "parent" in firewall_policy else "projects"
if parent_type == "projects" and parent not in config["monitored_projects"]:
continue
metric_labels = {'parent': parent, 'parent_type': parent_type}
metric_labels["name"] = firewall_policy[

131
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py
View File

@ -14,7 +14,9 @@
# limitations under the License.
#
from ast import AnnAssign
import re
from sqlite3 import Timestamp
import time
from collections import defaultdict
@ -80,8 +82,8 @@ def get_routes_for_network(config, network_link, project_id, routers_dict):
def get_dynamic_routes(config, metrics_dict, limits_dict):
'''
Writes all dynamic routes per VPC to custom metrics.
This function gets the usage, limit and utilization for the dynamic routes per VPC
note: assumes global routing is ON for all VPCs
Parameters:
config (dict): The dict containing config like clients and limits
metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
@ -130,10 +132,10 @@ def get_dynamic_routes(config, metrics_dict, limits_dict):
return dynamic_routes_dict
def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict):
def get_routes_ppg(config, metric_dict, usage_dict, limit_dict):
'''
This function gets the usage, limit and utilization for the dynamic routes per VPC peering group.
This function gets the usage, limit and utilization for the static or dynamic routes per VPC peering group.
note: assumes global routing is ON for all VPCs for dynamic routes, assumes share custom routes is on for all peered networks
Parameters:
config (dict): The dict containing config like clients and limits
metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
@ -142,11 +144,11 @@ def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict):
Returns:
None
'''
for project in config["monitored_projects"]:
network_dict_list = peerings.gather_peering_data(config, project)
for project_id in config["monitored_projects"]:
network_dict_list = peerings.gather_peering_data(config, project_id)
for network_dict in network_dict_list:
network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}"
network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}"
limit = limits.get_ppg(network_link, limit_dict)
@ -171,27 +173,23 @@ def get_dynamic_routes_ppg(config, metric_dict, usage_dict, limit_dict):
peered_network_dict["usage"] = peered_usage
peered_network_dict["limit"] = peered_limit
limits.count_effective_limit(config, project, network_dict,
limits.count_effective_limit(config, project_id, network_dict,
metric_dict["usage"]["name"],
metric_dict["limit"]["name"],
metric_dict["utilization"]["name"],
limit_dict)
print(
f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}"
f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}"
)
def get_static_routes_vpc(config, metrics_dict, project_quotas_dict):
def get_static_routes_dict(config):
'''
LOREM
Parameters:
config (dict): The dict containing config like clients and limits
metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value
project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
Returns:
None
Calls the Asset Inventory API to get all static customr routes under the GCP organization.
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
routes_per_vpc_dict (dictionary of string: int): Keys are the network links and values are the number of custom static routes per network.
'''
routes_per_vpc_dict = defaultdict()
usage_dict = defaultdict()
@ -203,10 +201,9 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict):
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ["compute.googleapis.com/Route"],
"read_mask": read_mask,
"read_mask": read_mask
})
timestamp = time.time()
for resource in response:
for versioned in resource.versioned_resources:
static_route = dict()
@ -216,30 +213,64 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict):
static_route["network"]).group(1)
static_route["network_name"] = re.search("\/([^\/]*)$",
static_route["network"]).group(1)
#exclude default vpc and peering routes
network_link = f"https://www.googleapis.com/compute/v1/projects/{static_route['project_id']}/global/networks/{static_route['network_name']}"
#exclude default vpc and peering routes, dynamic routes are not in Cloud Asset Inventory
if "nextHopPeering" not in static_route and "nextHopNetwork" not in static_route:
if static_route["project_id"] not in routes_per_vpc_dict:
routes_per_vpc_dict[static_route["project_id"]] = dict()
if static_route["network_name"] not in routes_per_vpc_dict[
static_route["project_id"]]:
routes_per_vpc_dict[static_route["project_id"]][
static_route["network_name"]] = dict()
if network_link not in routes_per_vpc_dict:
routes_per_vpc_dict[network_link] = dict()
routes_per_vpc_dict[network_link]["project_id"] = static_route[
"project_id"]
routes_per_vpc_dict[network_link]["network_name"] = static_route[
"network_name"]
if static_route["destRange"] not in routes_per_vpc_dict[network_link]:
routes_per_vpc_dict[network_link][static_route["destRange"]] = {}
if "usage" not in routes_per_vpc_dict[network_link]:
routes_per_vpc_dict[network_link]["usage"] = 0
routes_per_vpc_dict[network_link][
"usage"] = routes_per_vpc_dict[network_link]["usage"] + 1
if static_route["destRange"] not in routes_per_vpc_dict[
static_route["project_id"]][static_route["network_name"]]:
routes_per_vpc_dict[static_route["project_id"]][
static_route["network_name"]][static_route["destRange"]] = {}
if "usage" not in routes_per_vpc_dict[static_route["project_id"]][
static_route["network_name"]]:
routes_per_vpc_dict[static_route["project_id"]][
static_route["network_name"]]["usage"] = 0
routes_per_vpc_dict[static_route["project_id"]][
static_route["network_name"]]["usage"] = routes_per_vpc_dict[
static_route["project_id"]][
static_route["network_name"]]["usage"] + 1
#output a dict with network links and usage only
return {
network_link_out: routes_per_vpc_dict[network_link_out]["usage"]
for network_link_out in routes_per_vpc_dict
}
for project_id in routes_per_vpc_dict:
def get_static_routes_data(config, metrics_dict, static_routes_dict,
project_quotas_dict):
'''
Determines and writes the number of static routes for each VPC in monitored projects, the per project limit and the per project utilization
note: assumes custom routes sharing is ON for all VPCs
Parameters:
config (dict): The dict containing config like clients and limits
metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
static_routes_dict (dictionary of dictionary: int): Keys are the network links and values are the number of custom static routes per network.
project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
Returns:
None
'''
timestamp = time.time()
project_usage = {project: 0 for project in config["monitored_projects"]}
#usage is drilled down by network
for network_link in static_routes_dict:
project_id = re.search("\/([^\/]*)(\/[^\/]*){3}$", network_link).group(1)
if (project_id not in config["monitored_projects"]):
continue
network_name = re.search("\/([^\/]*)$", network_link).group(1)
project_usage[project_id] = project_usage[project_id] + static_routes_dict[
network_link]
metric_labels = {"project": project_id, "network_name": network_name}
metrics.append_data_to_series_buffer(
config, metrics_dict["metrics_per_network"]["static_routes_per_network"]
["usage"]["name"], static_routes_dict[network_link], metric_labels,
timestamp=timestamp)
#limit and utilization are calculated by projec
for project_id in project_usage:
current_quota_limit = project_quotas_dict[project_id]['global']["routes"][
"limit"]
if current_quota_limit is None:
@ -247,14 +278,6 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict):
f"Could not determine static routes metric for projects/{project_id} due to missing quotas"
)
continue
for network_name in routes_per_vpc_dict[project_id]:
metric_labels = {"project": project_id, "network_name": network_name}
metrics.append_data_to_series_buffer(
config, metrics_dict["metrics_per_network"]
["static_routes_per_network"]["usage"]["name"],
routes_per_vpc_dict[project_id][network_name]["usage"], metric_labels,
timestamp=timestamp)
# limit and utilization are calculted by project
metric_labels = {"project": project_id}
metrics.append_data_to_series_buffer(
@ -264,7 +287,7 @@ def get_static_routes_vpc(config, metrics_dict, project_quotas_dict):
metrics.append_data_to_series_buffer(
config, metrics_dict["metrics_per_network"]["static_routes_per_network"]
["utilization"]["name"],
routes_per_vpc_dict[project_id][network_name]["usage"] /
current_quota_limit, metric_labels, timestamp=timestamp)
project_usage[project_id] / current_quota_limit, metric_labels,
timestamp=timestamp)
return
return