zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'GoogleCloudPlatform:master' into updates-quota-monitoring-function

This commit is contained in:
maunope 2022-09-30 11:08:49 +02:00 committed by GitHub
parent eea4642cf2 a709febfdb
commit 9a67546cb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 547 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
blueprints/cloud-operations/network-dashboard/README.md
View File

@ -3,7 +3,7 @@
This repository provides an end-to-end solution to gather some GCP Networking quotas and limits (that cannot be seen in the GCP console today) and display them in a dashboard.
The goal is to allow for better visibility of these limits, facilitating capacity planning and avoiding hitting these limits.
Here is an blueprint of dashboard you can get with this solution:
Here is an example of dashboard you can get with this solution:
<img src="metric.png" width="640px">
@ -15,10 +15,11 @@ Here you see utilization (usage compared to the limit) for a specific metric (nu
Clone this repository, then go through the following steps to create resources:
- Create a terraform.tfvars file with the following content:
- organization_id = "[YOUR-ORG-ID]"
- billing_account = "[YOUR-BILLING-ACCOUNT]"
- organization_id = "<YOUR-ORG-ID>"
- billing_account = "<YOUR-BILLING-ACCOUNT>"
- monitoring_project_id = "project-0" # Monitoring project where the dashboard will be created and the solution deployed
- monitored_projects_list = ["project-1", "project2"] # Projects to be monitored by the solution
- monitored_folders_list = ["folder_id"] # Folders to be monitored by the solution
- `terraform init`
- `terraform apply`
@ -43,18 +44,17 @@ The Cloud Function currently tracks usage, limit and utilization of:
- internal forwarding rules for internal L7 load balancers per VPC peering group
- Dynamic routes per VPC
- Dynamic routes per VPC peering group
- IP utilization per subnet (% of IP addresses used in a subnet)
It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring.
Note that metrics are created in the cloud-function/metrics.yaml file.
You can also edit default limits for a specific network in that file. See the blueprint for `vpc_peering_per_network`.
You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`.
## Next steps and ideas
In a future release, we could support:
- Static routes per VPC / per VPC peering group
- Dynamic routes per VPC peering group
- Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore)
- Subnet IP ranges utilization
If you are interested in this and/or would like to contribute, please contact legranda@google.com.
If you are interested in this and/or would like to contribute, please contact legranda@google.com.

64
blueprints/cloud-operations/network-dashboard/cloud-function/main.py
View File

@ -14,23 +14,63 @@
# limitations under the License.
#
from code import interact
from distutils.command.config import config
import os
from pickletools import int4
import time
from google.cloud import monitoring_v3, asset_v1
from google.protobuf import field_mask_pb2
from googleapiclient import discovery
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes
from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, subnets
def get_monitored_projects_list(config):
'''
Gets the projects to be monitored from the MONITORED_FOLDERS_LIST environment variable.
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
monitored_projects (List of strings): Full list of projects to be monitored
'''
monitored_projects = config["monitored_projects"]
monitored_folders = os.environ.get("MONITORED_FOLDERS_LIST").split(",")
# Handling empty monitored folders list
if monitored_folders == ['']:
monitored_folders = []
# Gets all projects under each monitored folder (and even in sub folders)
for folder in monitored_folders:
read_mask = field_mask_pb2.FieldMask()
read_mask.FromJsonString('name,versionedResources')
response = config["clients"]["asset_client"].search_all_resources(
request={
"scope": f"folders/{folder}",
"asset_types": ["cloudresourcemanager.googleapis.com/Project"],
"read_mask": read_mask
})
for resource in response:
for versioned in resource.versioned_resources:
for field_name, field_value in versioned.resource.items():
if field_name == "projectId":
project_id = field_value
# Avoid duplicate
if project_id not in monitored_projects:
monitored_projects.append(project_id)
print("List of projects to be monitored:")
print(monitored_projects)
return monitored_projects
def monitoring_interval():
'''
Creates the monitoring interval of 24 hours
Returns:
monitoring_v3.TimeInterval: Moinitoring time interval of 24h
monitoring_v3.TimeInterval: Monitoring time interval of 24h
'''
now = time.time()
seconds = int(now)
@ -78,20 +118,25 @@ config = {
"discovery_client": discovery.build('compute', 'v1'),
"asset_client": asset_v1.AssetServiceClient(),
"monitoring_client": monitoring_v3.MetricServiceClient()
}
},
}
def main(event, context):
'''
Cloud Function Entry point, called by the scheduler.
Parameters:
event: Not used for now (Pubsub trigger)
context: Not used for now (Pubsub trigger)
Returns:
'Function executed successfully'
'''
# Handling empty monitored projects list
if config["monitored_projects"] == ['']:
config["monitored_projects"] = []
# Gets projects and folders to be monitored
config["monitored_projects"] = get_monitored_projects_list(config)
# Keep the monitoring interval up2date during each run
config["monitoring_interval"] = monitoring_interval()
@ -99,6 +144,9 @@ def main(event, context):
metrics_dict, limits_dict = metrics.create_metrics(
config["monitoring_project_link"])
# IP utilization subnet level metrics
subnets.get_subnets(config, metrics_dict)
# Asset inventory queries
gce_instance_dict = instances.get_gce_instance_dict(config)
l4_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L4")
@ -143,7 +191,7 @@ def main(event, context):
routes.get_dynamic_routes_ppg(
config, metrics_dict["metrics_per_peering_group"]
["dynamic_routes_per_peering_group"], dynamic_routes_dict,
limits_dict['number_of_subnet_IP_ranges_ppg_limit'])
limits_dict['dynamic_routes_per_peering_group_limit'])
return 'Function executed successfully'

17
blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml
View File

@ -14,6 +14,17 @@
# limitations under the License.
#
---
metrics_per_subnet:
ip_usage_per_subnet:
usage:
name: number_of_ip_used
description: Number of used IP addresses in the subnet.
utilization:
name: ip_addresses_per_subnet_utilization
description: Percentage of IP used in the subnet.
limit:
name: number_of_max_ip
description: Number of available IP addresses in the subnet.
metrics_per_network:
instance_per_network:
usage:
@ -60,7 +71,7 @@ metrics_per_network:
name: internal_forwarding_rules_l4_limit
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - limit.
values:
default_value: 300
default_value: 500
utilization:
name: internal_forwarding_rules_l4_utilization
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization.
@ -97,7 +108,7 @@ metrics_per_peering_group:
name: internal_forwarding_rules_l4_ppg_limit
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - effective limit.
values:
default_value: 300
default_value: 500
utilization:
name: internal_forwarding_rules_l4_ppg_utilization
description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - utilization.
@ -148,4 +159,4 @@ metrics_per_peering_group:
default_value: 300
utilization:
name: dynamic_routes_per_peering_group_utilization
description: Number of Dynamic routes per peering group - utilization.
description: Number of Dynamic routes per peering group - utilization.

20
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py
View File

@ -24,7 +24,6 @@ from . import peerings, limits, networks
def create_metrics(monitoring_project):
'''
Creates all Cloud Monitoring custom metrics based on the metric.yaml file
Parameters:
monitoring_project (string): the project where the metrics are written to
Returns:
@ -42,15 +41,16 @@ def create_metrics(monitoring_project):
metrics_dict = yaml.safe_load(stream)
for metric_list in metrics_dict.values():
for metric in metric_list.values():
for metric_name, metric in metric_list.items():
for sub_metric_key, sub_metric in metric.items():
metric_link = f"custom.googleapis.com/{sub_metric['name']}"
# If the metric doesn't exist yet, then we create it
if metric_link not in existing_metrics:
create_metric(sub_metric["name"], sub_metric["description"],
monitoring_project)
# Parse limits (both default values and network specific ones)
if sub_metric_key == "limit":
# Parse limits for network and peering group metrics
# Subnet level metrics have a different limit: the subnet IP range size
if sub_metric_key == "limit" and metric_name != "ip_usage_per_subnet":
limits_dict_for_metric = {}
for network_link, limit_value in sub_metric["values"].items():
limits_dict_for_metric[network_link] = limit_value
@ -64,7 +64,6 @@ def create_metrics(monitoring_project):
def create_metric(metric_name, description, monitoring_project):
'''
Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing
Parameters:
metric_name (string): Name of the metric to be created
description (string): Description of the metric to be created
@ -85,16 +84,16 @@ def create_metric(metric_name, description, monitoring_project):
def write_data_to_metric(config, monitored_project_id, value, metric_name,
network_name):
network_name, subnet_id=None):
'''
Writes data to Cloud Monitoring custom metrics.
Parameters:
config (dict): The dict containing config like clients and limits
monitored_project_id: ID of the project where the resource lives (will be added as a label)
value (int): Value for the data point of the metric.
metric_name (string): Name of the metric
network_name (string): Name of the network (will be added as a label)
subnet_id (string): Identifier of the Subnet (region/name of the subnet)
Returns:
usage (int): Current usage for that network.
limit (int): Current usage for that network.
@ -106,6 +105,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name,
series.resource.type = "global"
series.metric.labels["network_name"] = network_name
series.metric.labels["project"] = monitored_project_id
if subnet_id:
series.metric.labels["subnet_id"] = subnet_id
now = time.time()
seconds = int(now)
@ -129,13 +130,13 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name,
client.create_time_series(name=config["monitoring_project_link"],
time_series=[series])
except Exception as e:
print("Error while writing data point for metric", metric_name)
print(e)
def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict):
'''
This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored.
Parameters:
config (dict): The dict containing config like clients and limits
metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
@ -221,7 +222,6 @@ def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict):
def customize_quota_view(quota_results):
'''
Customize the quota output for an easier parsable output.
Parameters:
quota_results (string): Input from get_quota_current_usage or get_quota_current_limit. Contains the Current usage or limit for all networks in that project.
Returns:
@ -235,4 +235,4 @@ def customize_quota_view(quota_results):
for val in result.points:
quotaViewJson.update({'value': val.value.int64_value})
quotaViewList.append(quotaViewJson)
return quotaViewList
return quotaViewList

253
blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py Normal file
View File

@ -0,0 +1,253 @@
#
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from . import metrics
from google.protobuf import field_mask_pb2
from google.protobuf.json_format import MessageToDict
import ipaddress
def get_all_subnets(config):
'''
Returns a dictionary with subnet level informations (such as IP utilization)
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
subnet_dict (dictionary of String: dictionary): Key is the project_id, value is a nested dictionary with subnet_region/subnet_name as the key.
'''
subnet_dict = {}
read_mask = field_mask_pb2.FieldMask()
read_mask.FromJsonString('name,versionedResources')
response = config["clients"]["asset_client"].search_all_resources(
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ['compute.googleapis.com/Subnetwork'],
"read_mask": read_mask,
})
for asset in response:
for versioned in asset.versioned_resources:
subnet_name = ""
network_name = ""
project_id = ""
ip_cidr_range = ""
subnet_region = ""
for field_name, field_value in versioned.resource.items():
if field_name == 'name':
subnet_name = field_value
elif field_name == 'network':
# Network self link format:
# "https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/global/networks/<NETWORK_NAME>"
project_id = field_value.split('/')[6]
network_name = field_value.split('/')[-1]
elif field_name == 'ipCidrRange':
ip_cidr_range = field_value
elif field_name == 'region':
subnet_region = field_value.split('/')[-1]
net = ipaddress.ip_network(ip_cidr_range)
# Note that 4 IP addresses are reserved by GCP in all subnets
# Source: https://cloud.google.com/vpc/docs/subnets#reserved_ip_addresses_in_every_subnet
total_ip_addresses = int(net.num_addresses) - 4
if project_id not in subnet_dict:
subnet_dict[project_id] = {}
subnet_dict[project_id][f"{subnet_region}/{subnet_name}"] = {
'name': subnet_name,
'region': subnet_region,
'ip_cidr_range': ip_cidr_range,
'total_ip_addresses': total_ip_addresses,
'used_ip_addresses': 0,
'network_name': network_name
}
return subnet_dict
def compute_subnet_utilization(config, all_subnets_dict):
'''
Counts resources (VMs, ILBs, reserved IPs) using private IPs in the different subnets.
Parameters:
config (dict): Dict containing config like clients and limits
all_subnets_dict (dict): Dict containing the information for each subnets in the GCP organization
Returns:
None
'''
read_mask = field_mask_pb2.FieldMask()
read_mask.FromJsonString('name,versionedResources')
response_vm = config["clients"]["asset_client"].search_all_resources(
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ["compute.googleapis.com/Instance"],
"read_mask": read_mask,
})
# Counting IP addresses for GCE instances (VMs)
for asset in response_vm:
for versioned in asset.versioned_resources:
for field_name, field_value in versioned.resource.items():
if field_name == 'networkInterfaces':
response_dict = MessageToDict(list(field_value._pb)[0])
# Subnet self link:
# https://www.googleapis.com/compute/v1/projects/<project_id>/regions/<subnet_region>/subnetworks/<subnet_name>
subnet_region = response_dict['subnetwork'].split('/')[-3]
subnet_name = response_dict['subnetwork'].split('/')[-1]
# Network self link:
# https://www.googleapis.com/compute/v1/projects/<project_id>/global/networks/<network_name>
project_id = response_dict['network'].split('/')[6]
network_name = response_dict['network'].split('/')[-1]
all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][
'used_ip_addresses'] += 1
response_ilb = config["clients"]["asset_client"].search_all_resources(
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ["compute.googleapis.com/ForwardingRule"],
"read_mask": read_mask,
})
# Counting IP addresses for GCE Internal Load Balancers
for asset in response_ilb:
internal = False
psc = False
project_id = ''
subnet_name = ''
subnet_region = ''
address = ''
for versioned in asset.versioned_resources:
for field_name, field_value in versioned.resource.items():
if 'loadBalancingScheme' in field_name and field_value == 'INTERNAL':
internal = True
# We want to count only accepted PSC endpoint Forwarding Rule
# If the PSC endpoint Forwarding Rule is pending, we will count it in the reserved IP addresses
elif field_name == 'pscConnectionStatus' and field_value == 'ACCEPTED':
psc = True
elif field_name == 'IPAddress':
address = field_value
elif field_name == 'network':
project_id = field_value.split('/')[6]
elif 'subnetwork' in field_name:
subnet_name = field_value.split('/')[-1]
subnet_region = field_value.split('/')[-3]
if internal:
all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][
'used_ip_addresses'] += 1
elif psc:
# PSC endpoint asset doesn't contain the subnet information in Asset Inventory
# We need to find the correct subnet with IP address matching
ip_address = ipaddress.ip_address(address)
for subnet_key, subnet_dict in all_subnets_dict[project_id].items():
if ip_address in ipaddress.ip_network(subnet_dict['ip_cidr_range']):
all_subnets_dict[project_id][subnet_key]['used_ip_addresses'] += 1
response_reserved_ips = config["clients"][
"asset_client"].search_all_resources(
request={
"scope": f"organizations/{config['organization']}",
"asset_types": ["compute.googleapis.com/Address"],
"read_mask": read_mask,
})
# Counting IP addresses for GCE Reserved IPs (ex: PSC, Cloud DNS Inbound policies, reserved GCE IPs)
for asset in response_reserved_ips:
purpose = ""
status = ""
project_id = ""
network_name = ""
subnet_name = ""
subnet_region = ""
address = ""
prefixLength = ""
for versioned in asset.versioned_resources:
for field_name, field_value in versioned.resource.items():
if field_name == 'purpose':
purpose = field_value
elif field_name == 'region':
subnet_region = field_value.split('/')[-1]
elif field_name == 'status':
status = field_value
elif field_name == 'address':
address = field_value
elif field_name == 'network':
network_name = field_value.split('/')[-1]
project_id = field_value.split('/')[6]
elif field_name == 'subnetwork':
subnet_name = field_value.split('/')[-1]
project_id = field_value.split('/')[6]
elif field_name == 'prefixLength':
prefixLength = field_value
# Rserved IP addresses for GCE instances or PSC Forwarding Rule PENDING state
if purpose == "GCE_ENDPOINT" and status == "RESERVED":
all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][
'used_ip_addresses'] += 1
# Cloud DNS inbound policy
elif purpose == "DNS_RESOLVER":
all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][
'used_ip_addresses'] += 1
# PSA Range for Cloud SQL, MemoryStore, etc.
elif purpose == "VPC_PEERING":
# TODO: PSA range to be handled later
# print("PSA range to be handled later:", address, prefixLength, network_name)
continue
def get_subnets(config, metrics_dict):
'''
Writes all subnet metrics to custom metrics.
Parameters:
config (dict): The dict containing config like clients and limits
Returns:
None
'''
all_subnets_dict = get_all_subnets(config)
# Updates all_subnets_dict with the IP utilization info
compute_subnet_utilization(config, all_subnets_dict)
for project_id in config["monitored_projects"]:
if project_id not in all_subnets_dict:
continue
for subnet_dict in all_subnets_dict[project_id].values():
ip_utilization = 0
if subnet_dict['used_ip_addresses'] > 0:
ip_utilization = subnet_dict['used_ip_addresses'] / subnet_dict[
'total_ip_addresses']
# Building unique identifier with subnet region/name
subnet_id = f"{subnet_dict['region']}/{subnet_dict['name']}"
metrics.write_data_to_metric(
config, project_id, subnet_dict['used_ip_addresses'],
metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"]["usage"]
["name"], subnet_dict['network_name'], subnet_id)
metrics.write_data_to_metric(
config, project_id, subnet_dict['total_ip_addresses'],
metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"]["limit"]
["name"], subnet_dict['network_name'], subnet_id)
metrics.write_data_to_metric(
config, project_id, ip_utilization, metrics_dict["metrics_per_subnet"]
["ip_usage_per_subnet"]["utilization"]["name"],
subnet_dict['network_name'], subnet_id)
print("Wrote metrics for subnet ip utilization for VPCs in project",
project_id)

309
blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json
View File

@ -4,383 +4,456 @@
"columns": 12,
"tiles": [
{
"width": 6,
"height": 4,
"widget": {
"title": "internal_forwarding_rules_l4_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "1800s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6
},
{
"yPos": 12,
"width": 6,
"height": 4,
"widget": {
"title": "internal_forwarding_rules_l7_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"yPos": 12
},
{
"yPos": 8,
"width": 6,
"height": 4,
"widget": {
"title": "number_of_instances_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"yPos": 8
},
{
"xPos": 6,
"yPos": 4,
"width": 6,
"height": 4,
"widget": {
"title": "number_of_vpc_peerings_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"xPos": 6,
"yPos": 4
},
{
"yPos": 4,
"width": 6,
"height": 4,
"widget": {
"title": "number_of_active_vpc_peerings_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_INTERPOLATE"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"yPos": 4
},
{
"yPos": 16,
"width": 6,
"height": 4,
"widget": {
"title": "subnet_IP_ranges_ppg_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"yPos": 16
},
{
"xPos": 6,
"width": 6,
"height": 4,
"widget": {
"title": "internal_forwarding_rules_l4_ppg_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"xPos": 6
},
{
"xPos": 6,
"yPos": 12,
"width": 6,
"height": 4,
"widget": {
"title": "internal_forwarding_rules_l7_ppg_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
},
"filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
}
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"xPos": 6,
"yPos": 12
},
{
"xPos": 6,
"yPos": 8,
"width": 6,
"height": 4,
"widget": {
"title": "number_of_instances_ppg_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "3600s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "3600s",
"perSeriesAligner": "ALIGN_NEXT_OLDER"
}
},
"filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\""
}
},
"plotType": "LINE",
"minAlignmentPeriod": "3600s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"xPos": 6,
"yPos": 8
},
{
"xPos": 6,
"yPos": 16,
"width": 6,
"height": 4,
"widget": {
"title": "dynamic_routes_per_network_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"",
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
},
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
"filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\""
}
},
"plotType": "LINE",
"minAlignmentPeriod": "60s",
"targetAxis": "Y1"
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
},
"chartOptions": {
"mode": "COLOR"
}
}
}
},
"width": 6,
"yPos": 20
},
{
"height": 4,
"widget": {
"title": "ip_addresses_per_subnet_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
},
"filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"xPos": 6,
"yPos": 16
},
{
"height": 4,
"widget": {
"title": "dynamic_routes_ppg_utilization",
"xyChart": {
"chartOptions": {
"mode": "COLOR"
},
"dataSets": [
{
"minAlignmentPeriod": "60s",
"plotType": "LINE",
"targetAxis": "Y1",
"timeSeriesQuery": {
"timeSeriesFilter": {
"aggregation": {
"alignmentPeriod": "60s",
"perSeriesAligner": "ALIGN_MEAN"
},
"filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_peering_group_utilization\" resource.type=\"global\"",
"secondaryAggregation": {
"alignmentPeriod": "60s"
}
}
}
}
],
"timeshiftDuration": "0s",
"yAxis": {
"label": "y1Axis",
"scale": "LINEAR"
}
}
},
"width": 6,
"xPos": 6,
"yPos": 20
}
]
}
},
"name": "projects/347834224817/dashboards/1bdcd06a-030d-4977-bf4b-f32231aa3b77"
}

12
blueprints/cloud-operations/network-dashboard/main.tf
View File

@ -15,8 +15,11 @@
*/
locals {
project_id_list = toset(var.monitored_projects_list)
projects = join(",", local.project_id_list)
project_ids = toset(var.monitored_projects_list)
projects = join(",", local.project_ids)
folder_ids = toset(var.monitored_folders_list)
folders = join(",", local.folder_ids)
monitoring_project = var.monitoring_project_id == "" ? module.project-monitoring[0].project_id : var.monitoring_project_id
}
@ -90,6 +93,7 @@ resource "google_cloud_scheduler_job" "job" {
}
}
module "cloud-function" {
source = "../../../modules/cloud-function"
project_id = local.monitoring_project
@ -116,11 +120,13 @@ module "cloud-function" {
environment_variables = {
MONITORED_PROJECTS_LIST = local.projects
MONITORED_FOLDERS_LIST = local.folders
MONITORING_PROJECT_ID = local.monitoring_project
ORGANIZATION_ID = var.organization_id
}
service_account = module.service-account-function.email
service_account = module.service-account-function.email
ingress_settings = "ALLOW_INTERNAL_ONLY"
trigger_config = {
event = "google.pubsub.topic.publish"

9
blueprints/cloud-operations/network-dashboard/variables.tf
View File

@ -32,15 +32,20 @@ variable "prefix" {
default = ""
}
# TODO: support folder instead of a list of projects?
variable "monitored_projects_list" {
type = list(string)
description = "ID of the projects to be monitored (where limits and quotas data will be pulled)"
}
variable "monitored_folders_list" {
type = list(string)
description = "ID of the projects to be monitored (where limits and quotas data will be pulled)"
default = []
}
variable "schedule_cron" {
description = "Cron format schedule to run the Cloud Function. Default is every 5 minutes."
default = "*/5 * * * *"
default = "*/10 * * * *"
}
variable "project_monitoring_services" {