Merge branch 'master' into lcaggio/gcs2bq_shared_vpc

examples/cloud-operations/network-dashboard/README.md

@@ -27,7 +27,6 @@ A dashboard called "quotas-utilization" should be created.
 
 The Cloud Function runs every 5 minutes by default so you should start getting some data points after a few minutes.
 You can change this frequency by modifying the "schedule_cron" variable in variables.tf.
-Note that we are using Google defined metrics that are populated only once a day so you might need to wait up to one day for some metrics.
 
 Once done testing, you can clean up resources by running `terraform destroy`.
 
@@ -42,5 +41,19 @@ The Cloud Function currently tracks usage, limit and utilization of:
 - internal forwarding rules for internal L7 load balancers per VPC
 - internal forwarding rules for internal L4 load balancers per VPC peering group
 - internal forwarding rules for internal L7 load balancers per VPC peering group
+- Dynamic routes per VPC
 
 It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring.
+
+Note that metrics are created in the cloud-function/metrics.yaml file.
+
+You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`.
+
+## Next steps and ideas
+In a future release, we could support:
+- Static routes per VPC / per VPC peering group
+- Dynamic routes per VPC peering group
+- Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore)
+- Subnet IP ranges utilization
+
+If you are interested in this and/or would like to contribute, please contact legranda@google.com.

examples/cloud-operations/network-dashboard/cloud-function/main.py

@@ -17,10 +17,15 @@
 import os
 import time
 import yaml
+from collections import defaultdict
 from google.api import metric_pb2 as ga_metric
-from google.cloud import monitoring_v3
+from google.api_core import protobuf_helpers
+from google.cloud import monitoring_v3, asset_v1
+from google.protobuf import field_mask_pb2
 from googleapiclient import discovery
 
+# Organization ID containing the projects to be monitored
+ORGANIZATION_ID = os.environ.get("ORGANIZATION_ID")
 # list of projects from which function will get quotas information
 MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",")
 # project where the metrics and dahsboards will be created
@@ -28,25 +33,11 @@ MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID")
 MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}"
 service = discovery.build('compute', 'v1')
 
-# DEFAULT LIMITS:
-LIMIT_INSTANCES = os.environ.get("LIMIT_INSTANCES").split(",")
-LIMIT_INSTANCES_PPG = os.environ.get("LIMIT_INSTANCES_PPG").split(",")
-LIMIT_L4 = os.environ.get("LIMIT_L4").split(",")
-LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",")
-LIMIT_L7 = os.environ.get("LIMIT_L7").split(",")
-LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",")
-LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",")
-LIMIT_VPC_PEER = os.environ.get("LIMIT_VPC_PEER").split(",")
-
 # Existing GCP metrics per network
 GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
-GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
 L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
-L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
 L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit"
-L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage"
 SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit"
-SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage"
 
 
 def main(event, context):
@@ -59,39 +50,203 @@ def main(event, context):
       Returns:
         'Function executed successfully'
   '''
+  metrics_dict, limits_dict = create_metrics()
 
-  metrics_dict = create_metrics()
+  # Asset inventory queries
+  gce_instance_dict = get_gce_instance_dict()
+  l4_forwarding_rules_dict = get_l4_forwarding_rules_dict()
+  l7_forwarding_rules_dict = get_l7_forwarding_rules_dict()
+  subnet_range_dict = get_subnet_ranges_dict()
 
   # Per Network metrics
-  get_gce_instances_data(metrics_dict)
+  get_gce_instances_data(metrics_dict, gce_instance_dict,
-  get_l4_forwarding_rules_data(metrics_dict)
+                         limits_dict['number_of_instances_limit'])
-  get_vpc_peering_data(metrics_dict)
+  get_l4_forwarding_rules_data(
+      metrics_dict, l4_forwarding_rules_dict,
+      limits_dict['internal_forwarding_rules_l4_limit'])
+  get_vpc_peering_data(metrics_dict,
+                       limits_dict['number_of_vpc_peerings_limit'])
+  get_dynamic_routes(metrics_dict,
+                     limits_dict['dynamic_routes_per_network_limit'])
 
+  # Per VPC peering group metrics
   get_pgg_data(
       metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"],
-      GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC,
+      gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC,
-      LIMIT_INSTANCES_PPG)
+      limits_dict['number_of_instances_ppg_limit'])
 
   get_pgg_data(
       metrics_dict["metrics_per_peering_group"]
-      ["l4_forwarding_rules_per_peering_group"],
+      ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict,
-      L4_FORWARDING_RULES_USAGE_METRIC, L4_FORWARDING_RULES_LIMIT_METRIC,
+      L4_FORWARDING_RULES_LIMIT_METRIC,
-      LIMIT_L4_PPG)
+      limits_dict['internal_forwarding_rules_l4_ppg_limit'])
 
   get_pgg_data(
       metrics_dict["metrics_per_peering_group"]
-      ["l7_forwarding_rules_per_peering_group"],
+      ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict,
-      L7_FORWARDING_RULES_USAGE_METRIC, L7_FORWARDING_RULES_LIMIT_METRIC,
+      L7_FORWARDING_RULES_LIMIT_METRIC,
-      LIMIT_L7_PPG)
+      limits_dict['internal_forwarding_rules_l7_ppg_limit'])
 
   get_pgg_data(
       metrics_dict["metrics_per_peering_group"]
-      ["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC,
+      ["subnet_ranges_per_peering_group"], subnet_range_dict,
-      SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS)
+      SUBNET_RANGES_LIMIT_METRIC,
+      limits_dict['number_of_subnet_IP_ranges_ppg_limit'])
 
   return 'Function executed successfully'
 
 
+def get_l4_forwarding_rules_dict():
+  '''
+    Calls the Asset Inventory API to get all L4 Forwarding Rules under the GCP organization.
+
+      Parameters:
+        None
+      Returns:
+        forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
+  '''
+  client = asset_v1.AssetServiceClient()
+
+  read_mask = field_mask_pb2.FieldMask()
+  read_mask.FromJsonString('name,versionedResources')
+
+  forwarding_rules_dict = defaultdict(int)
+
+  response = client.search_all_resources(
+      request={
+          "scope": f"organizations/{ORGANIZATION_ID}",
+          "asset_types": ["compute.googleapis.com/ForwardingRule"],
+          "read_mask": read_mask,
+      })
+  for resource in response:
+    internal = False
+    network_link = ""
+    for versioned in resource.versioned_resources:
+      for field_name, field_value in versioned.resource.items():
+        if field_name == "loadBalancingScheme":
+          internal = (field_value == "INTERNAL")
+        if field_name == "network":
+          network_link = field_value
+    if internal:
+      if network_link in forwarding_rules_dict:
+        forwarding_rules_dict[network_link] += 1
+      else:
+        forwarding_rules_dict[network_link] = 1
+
+  return forwarding_rules_dict
+
+
+def get_l7_forwarding_rules_dict():
+  '''
+    Calls the Asset Inventory API to get all L7 Forwarding Rules under the GCP organization.
+
+      Parameters:
+        None
+      Returns:
+        forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
+  '''
+  client = asset_v1.AssetServiceClient()
+
+  read_mask = field_mask_pb2.FieldMask()
+  read_mask.FromJsonString('name,versionedResources')
+
+  forwarding_rules_dict = defaultdict(int)
+
+  response = client.search_all_resources(
+      request={
+          "scope": f"organizations/{ORGANIZATION_ID}",
+          "asset_types": ["compute.googleapis.com/ForwardingRule"],
+          "read_mask": read_mask,
+      })
+  for resource in response:
+    internal = False
+    network_link = ""
+    for versioned in resource.versioned_resources:
+      for field_name, field_value in versioned.resource.items():
+        if field_name == "loadBalancingScheme":
+          internal = (field_value == "INTERNAL_MANAGED")
+        if field_name == "network":
+          network_link = field_value
+    if internal:
+      if network_link in forwarding_rules_dict:
+        forwarding_rules_dict[network_link] += 1
+      else:
+        forwarding_rules_dict[network_link] = 1
+
+  return forwarding_rules_dict
+
+
+def get_gce_instance_dict():
+  '''
+    Calls the Asset Inventory API to get all GCE instances under the GCP organization.
+
+      Parameters:
+        None
+      Returns:
+        gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network.
+  '''
+  client = asset_v1.AssetServiceClient()
+
+  gce_instance_dict = defaultdict(int)
+
+  response = client.search_all_resources(
+      request={
+          "scope": f"organizations/{ORGANIZATION_ID}",
+          "asset_types": ["compute.googleapis.com/Instance"],
+      })
+  for resource in response:
+    for field_name, field_value in resource.additional_attributes.items():
+      if field_name == "networkInterfaceNetworks":
+        for network in field_value:
+          if network in gce_instance_dict:
+            gce_instance_dict[network] += 1
+          else:
+            gce_instance_dict[network] = 1
+
+  return gce_instance_dict
+
+
+def get_subnet_ranges_dict():
+  '''
+    Calls the Asset Inventory API to get all Subnet ranges under the GCP organization.
+
+      Parameters:
+        None
+      Returns:
+        subnet_range_dict (dictionary of string: int): Keys are the network links and values are the number of subnet ranges per network.
+  '''
+  client = asset_v1.AssetServiceClient()
+  subnet_range_dict = defaultdict(int)
+  read_mask = field_mask_pb2.FieldMask()
+  read_mask.FromJsonString('name,versionedResources')
+
+  response = client.search_all_resources(
+      request={
+          "scope": f"organizations/{ORGANIZATION_ID}",
+          "asset_types": ["compute.googleapis.com/Subnetwork"],
+          "read_mask": read_mask,
+      })
+  for resource in response:
+    ranges = 0
+    network_link = None
+
+    for versioned in resource.versioned_resources:
+      for field_name, field_value in versioned.resource.items():
+        if field_name == "network":
+          network_link = field_value
+          ranges += 1
+        if field_name == "secondaryIpRanges":
+          for range in field_value:
+            ranges += 1
+
+    if network_link in subnet_range_dict:
+      subnet_range_dict[network_link] += ranges
+    else:
+      subnet_range_dict[network_link] = ranges
+
+  return subnet_range_dict
+
+
 def create_client():
   '''
     Creates the monitoring API client, that will be used to create, read and update custom metrics.
@@ -123,16 +278,41 @@ def create_client():
 
 
 def create_metrics():
+  '''
+    Creates all Cloud Monitoring custom metrics based on the metric.yaml file
+
+      Parameters:
+        None
+
+      Returns:
+        metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+        limits_dict (dictionary of dictionary of string: int): limits_dict[metric_name]: dict[network_name] = limit_value
+  '''
+  client = monitoring_v3.MetricServiceClient()
+  existing_metrics = []
+  for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK):
+    existing_metrics.append(desc.type)
+  limits_dict = {}
+
   with open("metrics.yaml", 'r') as stream:
     try:
       metrics_dict = yaml.safe_load(stream)
 
       for metric_list in metrics_dict.values():
         for metric in metric_list.values():
-          for sub_metric in metric.values():
+          for sub_metric_key, sub_metric in metric.items():
+            metric_link = f"custom.googleapis.com/{sub_metric['name']}"
+            # If the metric doesn't exist yet, then we create it
+            if metric_link not in existing_metrics:
               create_metric(sub_metric["name"], sub_metric["description"])
+            # Parse limits (both default values and network specific ones)
+            if sub_metric_key == "limit":
+              limits_dict_for_metric = {}
+              for network_link, limit_value in sub_metric["values"].items():
+                limits_dict_for_metric[network_link] = limit_value
+              limits_dict[sub_metric["name"]] = limits_dict_for_metric
 
-      return metrics_dict
+      return metrics_dict, limits_dict
     except yaml.YAMLError as exc:
       print(exc)
 
@@ -150,13 +330,6 @@ def create_metric(metric_name, description):
   '''
   client = monitoring_v3.MetricServiceClient()
 
-  metric_link = f"custom.googleapis.com/{metric_name}"
-  types = []
-  for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK):
-    types.append(desc.type)
-
-  # If the metric doesn't exist yet, then we create it
-  if metric_link not in types:
   descriptor = ga_metric.MetricDescriptor()
   descriptor.type = f"custom.googleapis.com/{metric_name}"
   descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE
@@ -167,59 +340,60 @@ def create_metric(metric_name, description):
   print("Created {}.".format(descriptor.name))
 
 
-def get_gce_instances_data(metrics_dict):
+def get_gce_instances_data(metrics_dict, gce_instance_dict, limit_dict):
   '''
     Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric.
 
       Parameters:
         metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+        gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network.
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
       Returns:
-        None
+        gce_instance_dict
   '''
   # Existing GCP Monitoring metrics for GCE instances
-  metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage"
   metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit"
 
   for project in MONITORED_PROJECTS_LIST:
     network_dict = get_networks(project)
 
-    current_quota_usage = get_quota_current_usage(f"projects/{project}",
-                                                  metric_instances_usage)
     current_quota_limit = get_quota_current_limit(f"projects/{project}",
                                                   metric_instances_limit)
-
-    current_quota_usage_view = customize_quota_view(current_quota_usage)
     current_quota_limit_view = customize_quota_view(current_quota_limit)
 
     for net in network_dict:
-      set_usage_limits(net, current_quota_usage_view, current_quota_limit_view,
+      set_limits(net, current_quota_limit_view, limit_dict)
-                       LIMIT_INSTANCES)
+
+      usage = 0
+      if net['self_link'] in gce_instance_dict:
+        usage = gce_instance_dict[net['self_link']]
+
       write_data_to_metric(
-          project, net['usage'], metrics_dict["metrics_per_network"]
+          project, usage, metrics_dict["metrics_per_network"]
-          ["instance_per_network"]["usage"]["name"], net['network name'])
+          ["instance_per_network"]["usage"]["name"], net['network_name'])
       write_data_to_metric(
           project, net['limit'], metrics_dict["metrics_per_network"]
-          ["instance_per_network"]["limit"]["name"], net['network name'])
+          ["instance_per_network"]["limit"]["name"], net['network_name'])
       write_data_to_metric(
-          project, net['usage'] / net['limit'],
+          project, usage / net['limit'], metrics_dict["metrics_per_network"]
-          metrics_dict["metrics_per_network"]["instance_per_network"]
+          ["instance_per_network"]["utilization"]["name"], net['network_name'])
-          ["utilization"]["name"], net['network name'])
 
     print(f"Wrote number of instances to metric for projects/{project}")
 
 
-def get_vpc_peering_data(metrics_dict):
+def get_vpc_peering_data(metrics_dict, limit_dict):
   '''
     Gets the data for VPC peerings (active or not) and writes it to the metric defined (vpc_peering_active_metric and vpc_peering_metric).
 
       Parameters:
         metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
       Returns:
         None
   '''
   for project in MONITORED_PROJECTS_LIST:
     active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(
-        project, LIMIT_VPC_PEER)
+        project, limit_dict)
     for peering in active_vpc_peerings:
       write_data_to_metric(
           project, peering['active_peerings'],
@@ -250,13 +424,13 @@ def get_vpc_peering_data(metrics_dict):
     print("Wrote number of VPC peerings to custom metric for project:", project)
 
 
-def gather_vpc_peerings_data(project_id, limit_list):
+def gather_vpc_peerings_data(project_id, limit_dict):
   '''
     Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric.
 
       Parameters:
         project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs.
-        limit_list (list of string): Used to get the limit per VPC or the default limit.
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
       Returns:
         active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering.
         peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering.
@@ -279,96 +453,100 @@ def gather_vpc_peerings_data(project_id, limit_list):
         peerings_count = 0
         active_peerings_count = 0
 
+      network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network['name']}"
+      network_limit = get_limit_ppg(network_link, limit_dict)
+
       active_d = {
           'project_id': project_id,
           'network_name': network['name'],
           'active_peerings': active_peerings_count,
-          'network_limit': get_limit(network['name'], limit_list)
+          'network_limit': network_limit
       }
       active_peerings_dict.append(active_d)
       d = {
           'project_id': project_id,
           'network_name': network['name'],
           'peerings': peerings_count,
-          'network_limit': get_limit(network['name'], limit_list)
+          'network_limit': network_limit
       }
       peerings_dict.append(d)
 
   return active_peerings_dict, peerings_dict
 
 
-def get_limit(network_name, limit_list):
+def get_limit_ppg(network_link, limit_dict):
   '''
     Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit.
 
       Parameters:
-        network_name (string): Name of the VPC network.
+        network_link (string): VPC network link.
         limit_list (list of string): Used to get the limit per VPC or the default limit.
       Returns:
-        limit (int): Limit for that VPC and that metric.
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
   '''
-  if network_name in limit_list:
+  if network_link in limit_dict:
-    return int(limit_list[limit_list.index(network_name) + 1])
+    return limit_dict[network_link]
   else:
-    if 'default_value' in limit_list:
+    if 'default_value' in limit_dict:
-      return int(limit_list[limit_list.index('default_value') + 1])
+      return limit_dict['default_value']
     else:
+      print(f"Error: limit not found for {network_link}")
       return 0
 
 
-def get_l4_forwarding_rules_data(metrics_dict):
+def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict,
+                                 limit_dict):
   '''
     Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric.
 
       Parameters:
-        metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions
+        metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
+        forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network.
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value.
       Returns:
         None
   '''
-  # Existing GCP Monitoring metrics for L4 Forwarding Rules
-  l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage"
-  l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit"
-
   for project in MONITORED_PROJECTS_LIST:
     network_dict = get_networks(project)
 
-    current_quota_usage = get_quota_current_usage(f"projects/{project}",
+    current_quota_limit = get_quota_current_limit(
-                                                  l4_forwarding_rules_usage)
+        f"projects/{project}", L4_FORWARDING_RULES_LIMIT_METRIC)
-    current_quota_limit = get_quota_current_limit(f"projects/{project}",
-                                                  l4_forwarding_rules_limit)
 
-    current_quota_usage_view = customize_quota_view(current_quota_usage)
     current_quota_limit_view = customize_quota_view(current_quota_limit)
 
     for net in network_dict:
-      set_usage_limits(net, current_quota_usage_view, current_quota_limit_view,
+      set_limits(net, current_quota_limit_view, limit_dict)
-                       LIMIT_L4)
+
+      usage = 0
+      if net['self_link'] in forwarding_rules_dict:
+        usage = forwarding_rules_dict[net['self_link']]
+
       write_data_to_metric(
-          project, net['usage'], metrics_dict["metrics_per_network"]
+          project, usage, metrics_dict["metrics_per_network"]
           ["l4_forwarding_rules_per_network"]["usage"]["name"],
-          net['network name'])
+          net['network_name'])
       write_data_to_metric(
           project, net['limit'], metrics_dict["metrics_per_network"]
           ["l4_forwarding_rules_per_network"]["limit"]["name"],
-          net['network name'])
+          net['network_name'])
       write_data_to_metric(
-          project, net['usage'] / net['limit'],
+          project, usage / net['limit'], metrics_dict["metrics_per_network"]
-          metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]
+          ["l4_forwarding_rules_per_network"]["utilization"]["name"],
-          ["utilization"]["name"], net['network name'])
+          net['network_name'])
 
     print(
         f"Wrote number of L4 forwarding rules to metric for projects/{project}")
 
 
-def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
+def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_dict):
   '''
     This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored.
 
       Parameters:
-        metric_dict (dictionary of string: string): A dictionary with the metric names and description, that will be used to populate the metrics
+        metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics
         usage_metric (string): Name of the existing GCP metric for usage per VPC network.
-        limit_metric (string): Name of the existing GCP metric for limit per VPC network.
+        usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value
-        limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit).
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
       Returns:
         None
   '''
@@ -382,46 +560,51 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg):
 
     # For each network in this GCP project
     for network_dict in network_dict_list:
-      current_quota_usage = get_quota_current_usage(f"projects/{project}",
+      network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}"
-                                                    usage_metric)
+
       current_quota_limit = get_quota_current_limit(f"projects/{project}",
                                                     limit_metric)
-
-      current_quota_usage_view = customize_quota_view(current_quota_usage)
       current_quota_limit_view = customize_quota_view(current_quota_limit)
+      limit = get_limit_network(network_dict, network_link,
+                                current_quota_limit_view, limit_dict)
+
+      usage = 0
+      if network_link in usage_dict:
+        usage = usage_dict[network_link]
 
-      usage, limit = get_usage_limit(network_dict, current_quota_usage_view,
-                                     current_quota_limit_view, limit_ppg)
       # Here we add usage and limit to the network dictionary
       network_dict["usage"] = usage
       network_dict["limit"] = limit
 
       # For every peered network, get usage and limits
-      for peered_network in network_dict['peerings']:
+      for peered_network_dict in network_dict['peerings']:
-        peering_project_usage = customize_quota_view(
+        peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}"
-            get_quota_current_usage(f"projects/{peered_network['project_id']}",
+        peered_usage = 0
-                                    usage_metric))
+        if peered_network_link in usage_dict:
-        peering_project_limit = customize_quota_view(
+          peered_usage = usage_dict[peered_network_link]
-            get_quota_current_limit(f"projects/{peered_network['project_id']}",
-                                    limit_metric))
 
-        usage, limit = get_usage_limit(peered_network, peering_project_usage,
+        peering_project_limit = customize_quota_view(
-                                       peering_project_limit, limit_ppg)
+            get_quota_current_limit(
+                f"projects/{peered_network_dict['project_id']}", limit_metric))
+
+        peered_limit = get_limit_network(peered_network_dict,
+                                         peered_network_link,
+                                         peering_project_limit, limit_dict)
         # Here we add usage and limit to the peered network dictionary
-        peered_network["usage"] = usage
+        peered_network_dict["usage"] = peered_usage
-        peered_network["limit"] = limit
+        peered_network_dict["limit"] = peered_limit
 
       count_effective_limit(project, network_dict, metric_dict["usage"]["name"],
                             metric_dict["limit"]["name"],
-                            metric_dict["utilization"]["name"], limit_ppg)
+                            metric_dict["utilization"]["name"], limit_dict)
       print(
-          f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}"
+          f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}"
       )
 
 
 def count_effective_limit(project_id, network_dict, usage_metric_name,
                           limit_metric_name, utilization_metric_name,
-                          limit_ppg):
+                          limit_dict):
   '''
     Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics.
     Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit
@@ -432,7 +615,7 @@ def count_effective_limit(project_id, network_dict, usage_metric_name,
         usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group.
         limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group.
         utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group.
-        limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit).
+        limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit).
       Returns:
         None
   '''
@@ -445,16 +628,20 @@ def count_effective_limit(project_id, network_dict, usage_metric_name,
   for peered_network in network_dict['peerings']:
     peering_group_usage += peered_network['usage']
 
+  network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}"
+
   # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit)
   limit_step1 = max(network_dict['limit'],
-                    get_limit(network_dict['network_name'], limit_ppg))
+                    get_limit_ppg(network_link, limit_dict))
 
   # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network
   limit_step2 = []
   for peered_network in network_dict['peerings']:
+    peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}"
+
     limit_step2.append(
         max(peered_network['limit'],
-            get_limit(peered_network['network_name'], limit_ppg)))
+            get_limit_ppg(peered_network_link, limit_dict)))
 
   # Calculates effective limit: Step 3: Find minimum from the list created by Step 2
   limit_step3 = min(limit_step2)
@@ -485,12 +672,156 @@ def get_networks(project_id):
   network_dict = []
   if 'items' in response:
     for network in response['items']:
-      NETWORK = network['name']
+      network_name = network['name']
-      ID = network['id']
+      network_id = network['id']
-      d = {'project_id': project_id, 'network name': NETWORK, 'network id': ID}
+      self_link = network['selfLink']
+      d = {
+          'project_id': project_id,
+          'network_name': network_name,
+          'network_id': network_id,
+          'self_link': self_link
+      }
       network_dict.append(d)
   return network_dict
 
+def get_dynamic_routes(metrics_dict, limits_dict):
+  '''
+    Writes all dynamic routes per VPC to custom metrics.
+
+      Parameters:
+        metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions.
+        limits_dict (dictionary of string: int): key is network link (or 'default_value') and value is the limit for that network
+      Returns:
+        None
+  '''
+  routers_dict = get_routers()
+
+  for project_id in MONITORED_PROJECTS_LIST:
+    network_dict = get_networks(project_id)
+
+    for network in network_dict:
+      sum_routes = get_routes_for_network(network['self_link'], project_id,
+                                          routers_dict)
+
+      if network['self_link'] in limits_dict:
+        limit = limits_dict[network['self_link']]
+      else:
+        if 'default_value' in limits_dict:
+          limit = limits_dict['default_value']
+        else:
+          print("Error: couldn't find limit for dynamic routes.")
+          break
+
+      utilization = sum_routes / limit
+
+      write_data_to_metric(
+          project_id, sum_routes, metrics_dict["metrics_per_network"]
+          ["dynamic_routes_per_network"]["usage"]["name"],
+          network['network_name'])
+      write_data_to_metric(
+          project_id, limit, metrics_dict["metrics_per_network"]
+          ["dynamic_routes_per_network"]["limit"]["name"],
+          network['network_name'])
+      write_data_to_metric(
+          project_id, utilization, metrics_dict["metrics_per_network"]
+          ["dynamic_routes_per_network"]["utilization"]["name"],
+          network['network_name'])
+
+    print("Wrote metrics for dynamic routes for VPCs in project", project_id)
+
+
+def get_routes_for_network(network_link, project_id, routers_dict):
+  '''
+    Returns a the number of dynamic routes for a given network
+
+      Parameters:
+        network_link (string): Network self link.
+        project_id (string): Project ID containing the network.
+        routers_dict (dictionary of string: list of string): Dictionary with key as network link and value as list of router links.
+      Returns:
+        sum_routes (int): Number of routes in that network.
+  '''
+  sum_routes = 0
+
+  if network_link in routers_dict:
+    for router_link in routers_dict[network_link]:
+      # Router link is using the following format:
+      # 'https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/routers/ROUTER_NAME'
+      start = router_link.find("/regions/") + len("/regions/")
+      end = router_link.find("/routers/")
+      router_region = router_link[start:end]
+      router_name = router_link.split('/routers/')[1]
+      routes = get_routes_for_router(project_id, router_region, router_name)
+
+      sum_routes += routes
+
+  return sum_routes
+
+
+def get_routes_for_router(project_id, router_region, router_name):
+  '''
+    Returns the same of dynamic routes learned by a specific Cloud Router instance
+
+      Parameters:
+        project_id (string): Project ID for the project containing the Cloud Router.
+        router_region (string): GCP region for the Cloud Router.
+        router_name (string): Cloud Router name.
+      Returns:
+        sum_routes (int): Number of dynamic routes learned by the Cloud Router.
+  '''
+  request = service.routers().getRouterStatus(project=project_id,
+                                              region=router_region,
+                                              router=router_name)
+  response = request.execute()
+
+  sum_routes = 0
+
+  if 'result' in response:
+    for peer in response['result']['bgpPeerStatus']:
+      sum_routes += peer['numLearnedRoutes']
+
+  return sum_routes
+
+
+def get_routers():
+  '''
+    Returns a dictionary of all Cloud Routers in the GCP organization.
+
+      Parameters:
+        None
+      Returns:
+        routers_dict (dictionary of string: list of string): Key is the network link and value is a list of router links.
+  '''
+  client = asset_v1.AssetServiceClient()
+
+  read_mask = field_mask_pb2.FieldMask()
+  read_mask.FromJsonString('name,versionedResources')
+
+  routers_dict = {}
+
+  response = client.search_all_resources(
+      request={
+          "scope": f"organizations/{ORGANIZATION_ID}",
+          "asset_types": ["compute.googleapis.com/Router"],
+          "read_mask": read_mask,
+      })
+  for resource in response:
+    network_link = None
+    router_link = None
+    for versioned in resource.versioned_resources:
+      for field_name, field_value in versioned.resource.items():
+        if field_name == "network":
+          network_link = field_value
+        if field_name == "selfLink":
+          router_link = field_value
+
+    if network_link in routers_dict:
+      routers_dict[network_link].append(router_link)
+    else:
+      routers_dict[network_link] = [router_link]
+
+  return routers_dict
+
 
 def gather_peering_data(project_id):
   '''
@@ -564,29 +895,6 @@ def get_network_id(project_id, network_name):
   return network_id
 
 
-def get_quota_current_usage(project_link, metric_name):
-  '''
-    Retrieves quota usage for a specific metric.
-
-      Parameters:
-        project_link (string): Project link.
-        metric_name (string): Name of the metric.
-      Returns:
-        results_list (list of string): Current usage.
-  '''
-  client, interval = create_client()
-
-  results = client.list_time_series(
-      request={
-          "name": project_link,
-          "filter": f'metric.type = "{metric_name}"',
-          "interval": interval,
-          "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL
-      })
-  results_list = list(results)
-  return (results_list)
-
-
 def get_quota_current_limit(project_link, metric_name):
   '''
     Retrieves limit for a specific metric.
@@ -630,98 +938,64 @@ def customize_quota_view(quota_results):
   return quotaViewList
 
 
-def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list):
+def set_limits(network_dict, quota_limit, limit_dict):
   '''
-    Updates the network dictionary with quota usage and limit values. 
+    Updates the network dictionary with quota limit values. 
 
       Parameters:
         network_dict (dictionary of string: string): Contains network information.
-        quota_usage (list of dictionaries of string: string): Current quota usage.
         quota_limit (list of dictionaries of string: string): Current quota limit.
-        limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit).
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
       Returns:
         None
   '''
-  if quota_usage:
+
-    for net in quota_usage:
+  network_dict['limit'] = None
-      if net['network_id'] == network_dict[
-          'network id']:  # if network ids in GCP quotas and in dictionary (using API) are the same
-        network_dict['usage'] = net['value']  # set network usage in dictionary
-        break
-      else:
-        network_dict['usage'] = 0  # if network does not appear in GCP quotas
-  else:
-    network_dict['usage'] = 0  # if quotas does not appear in GCP quotas
 
   if quota_limit:
     for net in quota_limit:
-      if net['network_id'] == network_dict[
+      if net['network_id'] == network_dict['network_id']:
-          'network id']:  # if network ids in GCP quotas and in dictionary (using API) are the same
+        network_dict['limit'] = net['value']
-        network_dict['limit'] = net['value']  # set network limit in dictionary
+        return
-        break
+
+  network_link = f"https://www.googleapis.com/compute/v1/projects/{network_dict['project_id']}/global/networks/{network_dict['network_name']}"
+
+  if network_link in limit_dict:
+    network_dict['limit'] = limit_dict[network_link]
   else:
-        if network_dict[
+    if 'default_value' in limit_dict:
-            'network name'] in limit_list:  # if network limit is in the environmental variables
+      network_dict['limit'] = limit_dict['default_value']
-          network_dict['limit'] = int(
-              limit_list[limit_list.index(network_dict['network name']) + 1])
     else:
-          network_dict['limit'] = int(
+      print(f"Error: Couldn't find limit for {network_link}")
-              limit_list[limit_list.index('default_value') +
+      network_dict['limit'] = 0
-                         1])  # set default value
-  else:  # if quotas does not appear in GCP quotas
-    if network_dict['network name'] in limit_list:
-      network_dict['limit'] = int(
-          limit_list[limit_list.index(network_dict['network name']) +
-                     1])  # ["default", 100, "networkname", 200]
-    else:
-      network_dict['limit'] = int(limit_list[limit_list.index('default_value') +
-                                             1])
 
 
-def get_usage_limit(network, quota_usage, quota_limit, limit_list):
+def get_limit_network(network_dict, network_link, quota_limit, limit_dict):
   '''
-    Returns usage and limit for a specific network and metric.
+    Returns limit for a specific network and metric, using the GCP quota metrics or the values in the yaml file if not found.
 
       Parameters:
         network_dict (dictionary of string: string): Contains network information.
-        quota_usage (list of dictionaries of string: string): Current quota usage for all networks in that project.
+        network_link (string): Contains network link
         quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project.
-        limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit).
+        limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value
       Returns:
-        usage (int): Current usage for that network.
+        limit (int): Current limit for that network.
-        limit (int): Current usage for that network.
   '''
-  usage = 0
-  limit = 0
-
-  if quota_usage:
-    for net in quota_usage:
-      if net['network_id'] == network[
-          'network_id']:  # if network ids in GCP quotas and in dictionary (using API) are the same
-        usage = net['value']  # set network usage in dictionary
-        break
-
   if quota_limit:
     for net in quota_limit:
-      if net['network_id'] == network[
+      if net['network_id'] == network_dict['network_id']:
-          'network_id']:  # if network ids in GCP quotas and in dictionary (using API) are the same
+        return net['value']
-        limit = net['value']  # set network limit in dictionary
-        break
-      else:
-        if network[
-            'network_name'] in limit_list:  # if network limit is in the environmental variables
-          limit = int(limit_list[limit_list.index(network['network_name']) + 1])
-        else:
-          limit = int(limit_list[limit_list.index('default_value') +
-                                 1])  # set default value
-  else:  # if quotas does not appear in GCP quotas
-    if network['network_name'] in limit_list:
-      limit = int(limit_list[limit_list.index(network['network_name']) +
-                             1])  # ["default", 100, "networkname", 200]
-    else:
-      limit = int(limit_list[limit_list.index('default_value') + 1])
 
-  return usage, limit
+  if network_link in limit_dict:
+    return limit_dict[network_link]
+  else:
+    if 'default_value' in limit_dict:
+      return limit_dict['default_value']
+    else:
+      print(f"Error: Couldn't find limit for {network_link}")
+
+  return 0
 
 
 def write_data_to_metric(monitored_project_id, value, metric_name,
@@ -762,4 +1036,10 @@ def write_data_to_metric(monitored_project_id, value, metric_name,
   })
   series.points = [point]
 
-  client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series])
+  # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error
+  # Implement exponential backoff retries?
+  try:
+    client.create_time_series(name=MONITORING_PROJECT_LINK,
+                              time_series=[series])
+  except Exception as e:
+    print(e)

examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml

@@ -22,6 +22,8 @@ metrics_per_network:
     limit:
       name: number_of_instances_limit
       description: Number of instances per VPC network - limit.
+      values:
+        default_value: 15000
     utilization:
       name: number_of_instances_utilization
       description: Number of instances per VPC network - utilization.
@@ -32,6 +34,8 @@ metrics_per_network:
     limit:
       name: number_of_active_vpc_peerings_limit
       description: Number of active VPC Peerings per VPC - limit.
+      values:
+        default_value: 25
     utilization:
       name: number_of_active_vpc_peerings_utilization
       description: Number of active VPC Peerings per VPC - utilization.
@@ -42,6 +46,9 @@ metrics_per_network:
     limit:
       name: number_of_vpc_peerings_limit
       description: Number of VPC Peerings per VPC - limit.
+      values:
+        default_value: 25
+        https://www.googleapis.com/compute/v1/projects/net-dash-test-host-prod/global/networks/vpc-prod: 40
     utilization:
       name: number_of_vpc_peerings_utilization
       description: Number of VPC Peerings per VPC - utilization.
@@ -52,6 +59,8 @@ metrics_per_network:
     limit:
       name: internal_forwarding_rules_l4_limit
       description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - limit.
+      values:
+        default_value: 75
     utilization:
       name: internal_forwarding_rules_l4_utilization
       description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization.
@@ -62,9 +71,23 @@ metrics_per_network:
     limit:
       name: internal_forwarding_rules_l7_limit
       description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per network - effective limit.
+      values:
+        default_value: 75
     utilization:
       name: internal_forwarding_rules_l7_utilization
       description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per Vnetwork - utilization.
+  dynamic_routes_per_network:
+    usage:
+      name: dynamic_routes_per_network_usage
+      description: Number of Dynamic routes per network - usage.
+    limit:
+      name: dynamic_routes_per_network_limit
+      description: Number of Dynamic routes per network - limit.
+      values:
+        default_value: 100
+    utilization:
+      name: dynamic_routes_per_network_utilization
+      description: Number of Dynamic routes per network - utilization.
 metrics_per_peering_group:
   l4_forwarding_rules_per_peering_group:
     usage:
@@ -73,6 +96,8 @@ metrics_per_peering_group:
     limit:
       name: internal_forwarding_rules_l4_ppg_limit
       description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - effective limit.
+      values:
+        default_value: 175
     utilization:
       name: internal_forwarding_rules_l4_ppg_utilization
       description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - utilization.
@@ -83,18 +108,22 @@ metrics_per_peering_group:
     limit:
       name: internal_forwarding_rules_l7_ppg_limit
       description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - effective limit.
+      values:
+        default_value: 175
     utilization:
       name: internal_forwarding_rules_l7_ppg_utilization
       description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - utilization.
   subnet_ranges_per_peering_group:
     usage:
-      name: number_of_subnet_IP_ranges_usage
+      name: number_of_subnet_IP_ranges_ppg_usage
       description: Number of Subnet Ranges per peering group - usage.
     limit:
-      name: number_of_subnet_IP_ranges_limit
+      name: number_of_subnet_IP_ranges_ppg_limit
       description: Number of Subnet Ranges per peering group - effective limit.
+      values:
+        default_value: 400
     utilization:
-      name: number_of_subnet_IP_ranges_utilization
+      name: number_of_subnet_IP_ranges_ppg_utilization
       description: Number of Subnet Ranges per peering group - utilization.
   instance_per_peering_group:
     usage:
@@ -103,6 +132,8 @@ metrics_per_peering_group:
     limit:
       name: number_of_instances_ppg_limit
       description: Number of instances per peering group - limit.
+      values:
+        default_value: 15500
     utilization:
       name: number_of_instances_ppg_utilization
       description: Number of instances per peering group - utilization.

examples/cloud-operations/network-dashboard/cloud-function/requirements.txt

@@ -7,3 +7,4 @@ google-cloud-monitoring==2.9.1
 oauth2client==4.1.3
 google-api-core==2.7.0
 PyYAML==6.0
+google-cloud-asset==3.8.1

examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json

@@ -4,347 +4,382 @@
     "columns": 12,
     "tiles": [
       {
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "internal_forwarding_rules_l4_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "1800s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6
+            "chartOptions": {
+              "mode": "COLOR"
+            }
+          }
+        }
       },
       {
+        "yPos": 12,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "internal_forwarding_rules_l7_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "60s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
+            "chartOptions": {
-        "xPos": 6
+              "mode": "COLOR"
+            }
+          }
+        }
       },
       {
+        "yPos": 8,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "number_of_instances_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "60s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
+            "chartOptions": {
-        "yPos": 8
+              "mode": "COLOR"
+            }
+          }
+        }
       },
       {
+        "xPos": 6,
+        "yPos": 4,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "number_of_vpc_peerings_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "60s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
+            "chartOptions": {
-        "xPos": 6,
+              "mode": "COLOR"
-        "yPos": 4
+            }
+          }
+        }
       },
       {
+        "yPos": 4,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "number_of_active_vpc_peerings_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "60s",
                       "perSeriesAligner": "ALIGN_INTERPOLATE"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
-        "yPos": 4
-      },
-      {
-        "height": 4,
-        "widget": {
-          "title": "number_of_subnet_IP_ranges_utilization",
-          "xyChart": {
             "chartOptions": {
               "mode": "COLOR"
+            }
+          }
+        }
       },
+      {
+        "yPos": 16,
+        "width": 6,
+        "height": 4,
+        "widget": {
+          "title": "subnet_IP_ranges_ppg_utilization",
+          "xyChart": {
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
+            "chartOptions": {
-        "xPos": 6,
+              "mode": "COLOR"
-        "yPos": 8
+            }
+          }
+        }
       },
       {
+        "xPos": 6,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "internal_forwarding_rules_l4_ppg_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
+            "chartOptions": {
-        "yPos": 12
+              "mode": "COLOR"
+            }
+          }
+        }
       },
       {
+        "xPos": 6,
+        "yPos": 12,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "internal_forwarding_rules_l7_ppg_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
                     },
-                    "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"",
                     "secondaryAggregation": {
                       "alignmentPeriod": "60s",
                       "perSeriesAligner": "ALIGN_MEAN"
                     }
                   }
-                }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "3600s",
+                "targetAxis": "Y1"
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
-            }
-          }
             },
-        "width": 6,
+            "chartOptions": {
-        "xPos": 6,
+              "mode": "COLOR"
-        "yPos": 12
+            }
+          }
+        }
       },
       {
+        "xPos": 6,
+        "yPos": 8,
+        "width": 6,
         "height": 4,
         "widget": {
           "title": "number_of_instances_ppg_utilization",
           "xyChart": {
-            "chartOptions": {
-              "mode": "COLOR"
-            },
             "dataSets": [
               {
-                "minAlignmentPeriod": "3600s",
-                "plotType": "LINE",
-                "targetAxis": "Y1",
                 "timeSeriesQuery": {
                   "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"",
                     "aggregation": {
                       "alignmentPeriod": "3600s",
                       "perSeriesAligner": "ALIGN_NEXT_OLDER"
+                    }
+                  }
                 },
-                    "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"",
+                "plotType": "LINE",
-                    "secondaryAggregation": {
+                "minAlignmentPeriod": "3600s",
-                      "alignmentPeriod": "60s"
+                "targetAxis": "Y1"
-                    }
-                  }
-                }
               }
             ],
             "timeshiftDuration": "0s",
             "yAxis": {
               "label": "y1Axis",
               "scale": "LINEAR"
+            },
+            "chartOptions": {
+              "mode": "COLOR"
+            }
           }
         }
       },
+      {
+        "xPos": 6,
+        "yPos": 16,
         "width": 6,
-        "yPos": 16
+        "height": 4,
+        "widget": {
+          "title": "dynamic_routes_per_network_utilization",
+          "xyChart": {
+            "dataSets": [
+              {
+                "timeSeriesQuery": {
+                  "timeSeriesFilter": {
+                    "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"",
+                    "aggregation": {
+                      "alignmentPeriod": "60s",
+                      "perSeriesAligner": "ALIGN_MEAN"
+                    },
+                    "secondaryAggregation": {
+                      "alignmentPeriod": "60s"
+                    }
+                  }
+                },
+                "plotType": "LINE",
+                "minAlignmentPeriod": "60s",
+                "targetAxis": "Y1"
+              }
+            ],
+            "timeshiftDuration": "0s",
+            "yAxis": {
+              "label": "y1Axis",
+              "scale": "LINEAR"
+            },
+            "chartOptions": {
+              "mode": "COLOR"
+            }
+          }
+        }
       }
     ]
   }

examples/cloud-operations/network-dashboard/main.tf

@@ -17,23 +17,6 @@
 locals {
   project_id_list = toset(var.monitored_projects_list)
   projects        = join(",", local.project_id_list)
-
-  limit_instances          = join(",", local.limit_instances_list)
-  limit_instances_list     = tolist(var.limit_instances)
-  limit_instances_ppg      = join(",", local.limit_instances_ppg_list)
-  limit_instances_ppg_list = tolist(var.limit_instances_ppg)
-  limit_l4                 = join(",", local.limit_l4_list)
-  limit_l4_list            = tolist(var.limit_l4)
-  limit_l4_ppg             = join(",", local.limit_l4_ppg_list)
-  limit_l4_ppg_list        = tolist(var.limit_l4_ppg)
-  limit_l7                 = join(",", local.limit_l7_list)
-  limit_l7_list            = tolist(var.limit_l7)
-  limit_l7_ppg             = join(",", local.limit_l7_ppg_list)
-  limit_l7_ppg_list        = tolist(var.limit_l7_ppg)
-  limit_subnets            = join(",", local.limit_subnets_list)
-  limit_subnets_list       = tolist(var.limit_subnets)
-  limit_vpc_peer           = join(",", local.limit_vpc_peer_list)
-  limit_vpc_peer_list      = tolist(var.limit_vpc_peer)
 }
 
 ################################################
@@ -66,6 +49,7 @@ module "service-account-function" {
     "${var.organization_id}" = [
       "roles/compute.networkViewer",
       "roles/monitoring.viewer",
+      "roles/cloudasset.viewer"
     ]
   }
 
@@ -129,16 +113,9 @@ module "cloud-function" {
   }
 
   environment_variables = {
-    LIMIT_INSTANCES         = local.limit_instances
-    LIMIT_INSTANCES_PPG     = local.limit_instances_ppg
-    LIMIT_L4                = local.limit_l4
-    LIMIT_L4_PPG            = local.limit_l4_ppg
-    LIMIT_L7                = local.limit_l7
-    LIMIT_L7_PPG            = local.limit_l7_ppg
-    LIMIT_SUBNETS           = local.limit_subnets
-    LIMIT_VPC_PEER          = local.limit_vpc_peer
     MONITORED_PROJECTS_LIST = local.projects
     MONITORING_PROJECT_ID   = module.project-monitoring.project_id
+    ORGANIZATION_ID         = var.organization_id
   }
 
   service_account = module.service-account-function.email

examples/cloud-operations/network-dashboard/tests/test.tf

@@ -270,7 +270,6 @@ resource "google_compute_instance" "test-vm-hub1" {
 }
 
 # Forwarding Rules
-
 resource "google_compute_forwarding_rule" "forwarding-rule-dev" {
   count      = 10
   name       = "forwarding-rule-dev${count.index}"

examples/cloud-operations/network-dashboard/variables.tf

@@ -76,68 +76,3 @@ variable "zone" {
   description = "Zone used to deploy vms"
   default     = "europe-west1-b"
 }
-
-variable "limit_l4" {
-  description = "Maximum number of forwarding rules for Internal TCP/UDP Load Balancing per network."
-  type        = list(string)
-  default = [
-    "default_value", "75",
-  ]
-}
-
-variable "limit_l7" {
-  description = "Maximum number of forwarding rules for Internal HTTP(S) Load Balancing per network."
-  type        = list(string)
-  default = [
-    "default_value", "75",
-  ]
-}
-
-variable "limit_subnets" {
-  description = "Maximum number of subnet IP ranges (primary and secondary) per peering group"
-  type        = list(string)
-  default = [
-    "default_value", "400",
-  ]
-}
-
-variable "limit_instances" {
-  description = "Maximum number of instances per network"
-  type        = list(string)
-  default = [
-    "default_value", "15000",
-  ]
-}
-
-variable "limit_instances_ppg" {
-  description = "Maximum number of instances per peering group."
-  type        = list(string)
-  default = [
-    "default_value", "15000",
-  ]
-}
-
-variable "limit_vpc_peer" {
-  description = "Maximum number of peering VPC peerings per network."
-  type        = list(string)
-  default = [
-    "default_value", "25",
-    "test-vpc", "40",
-  ]
-}
-
-variable "limit_l4_ppg" {
-  description = "Maximum number of forwarding rules for Internal TCP/UDP Load Balancing per network."
-  type        = list(string)
-  default = [
-    "default_value", "175",
-  ]
-}
-
-variable "limit_l7_ppg" {
-  description = "Maximum number of forwarding rules for Internal HTTP(S) Load Balancing per network."
-  type        = list(string)
-  default = [
-    "default_value", "175",
-  ]
-}

examples/serverless/api-gateway/function/package-lock.json

@@ -896,9 +896,9 @@
       }
     },
     "node_modules/minimist": {
-      "version": "1.2.5",
+      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
     },
     "node_modules/ms": {
       "version": "2.0.0",
@@ -2125,9 +2125,9 @@
       }
     },
     "minimist": {
-      "version": "1.2.5",
+      "version": "1.2.6",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
     },
     "ms": {
       "version": "2.0.0",

fast/stages/02-networking-vpn/vpn-spoke-prod.tf

@@ -82,7 +82,7 @@ module "landing-to-prod-ew4-vpn" {
   source           = "../../../modules/net-vpn-ha"
   project_id       = module.landing-project.project_id
   network          = module.landing-vpc.self_link
-  region           = "europe-west1"
+  region           = "europe-west4"
   name             = "vpn-to-prod-ew4"
   router_create    = true
   router_name      = "landing-vpn-ew4"
@@ -112,7 +112,7 @@ module "prod-to-landing-ew4-vpn" {
   source           = "../../../modules/net-vpn-ha"
   project_id       = module.prod-spoke-project.project_id
   network          = module.prod-spoke-vpc.self_link
-  region           = "europe-west1"
+  region           = "europe-west4"
   name             = "vpn-to-landing-ew4"
   router_create    = true
   router_name      = "prod-spoke-vpn-ew4"

modules/apigee-organization/README.md

@@ -110,6 +110,7 @@ module "apigee-organization" {
 | [apigee_envgroups](variables.tf#L22) | Apigee Environment Groups. | <code title="map&#40;object&#40;&#123;&#10;  environments &#61; list&#40;string&#41;&#10;  hostnames    &#61; list&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [apigee_environments](variables.tf#L31) | Apigee Environment Names. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
 | [authorized_network](variables.tf#L37) | VPC network self link (requires service network peering enabled (Used in Apigee X only). | <code>string</code> |  | <code>null</code> |
+| [billing_type](variables.tf#L75) | Billing type of the Apigee organization. | <code>string</code> |  | <code>null</code> |
 | [database_encryption_key](variables.tf#L43) | Cloud KMS key self link (e.g. `projects/foo/locations/us/keyRings/bar/cryptoKeys/baz`) used for encrypting the data that is stored and replicated across runtime instances (immutable, used in Apigee X only). | <code>string</code> |  | <code>null</code> |
 | [description](variables.tf#L49) | Description of the Apigee Organization. | <code>string</code> |  | <code>&#34;Apigee Organization created by tf module&#34;</code> |
 | [display_name](variables.tf#L55) | Display Name of the Apigee Organization. | <code>string</code> |  | <code>null</code> |

modules/apigee-organization/main.tf

@@ -31,6 +31,7 @@ resource "google_apigee_organization" "apigee_org" {
   display_name                         = var.display_name
   description                          = var.description
   runtime_type                         = var.runtime_type
+  billing_type                         = var.billing_type
   authorized_network                   = var.authorized_network
   runtime_database_encryption_key_name = var.database_encryption_key
 }

modules/apigee-organization/variables.tf

@@ -72,4 +72,8 @@ variable "runtime_type" {
   }
 }
 
-
+variable "billing_type" {
+  description = "Billing type of the Apigee organization."
+  type        = string
+  default     = null
+}

modules/data-catalog-policy-tag/outputs.tf

@@ -16,7 +16,7 @@
 
 output "tags" {
   description = "Policy Tags."
-  value       = { for k, v in google_data_catalog_policy_tag.default : v.id => v.name }
+  value       = { for k, v in google_data_catalog_policy_tag.default : k => v.id }
 }
 
 output "taxonomy_id" {

tests/modules/apigee_organization/fixture/main.tf

@@ -19,6 +19,7 @@ module "test" {
   project_id         = "my-project"
   analytics_region   = var.analytics_region
   runtime_type       = "CLOUD"
+  billing_type       = "EVALUATION"
   authorized_network = var.network
   apigee_environments = [
     "eval1",