From 9f3ee4dc2299a7e9034cf25988d9c4b59c0fd70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Tue, 8 Mar 2022 18:36:02 +0100 Subject: [PATCH 01/23] Networking dashboard to display per VPC and per VPC peering group limits that are not shown in the console --- .../network-dashboard/LICENSE | 201 ++++++ .../network-dashboard/README.md | 37 ++ .../network-dashboard/cloud-function/main.py | 588 ++++++++++++++++++ .../cloud-function/requirements.txt | 8 + .../dashboards/quotas-utilization.json | 351 +++++++++++ .../network-dashboard/main.tf | 183 ++++++ .../network-dashboard/tests/README.md | 1 + .../network-dashboard/tests/test.tf | 279 +++++++++ .../network-dashboard/tests/variables.tf | 49 ++ .../network-dashboard/variables.tf | 150 +++++ 10 files changed, 1847 insertions(+) create mode 100644 examples/cloud-operations/network-dashboard/LICENSE create mode 100644 examples/cloud-operations/network-dashboard/README.md create mode 100644 examples/cloud-operations/network-dashboard/cloud-function/main.py create mode 100644 examples/cloud-operations/network-dashboard/cloud-function/requirements.txt create mode 100644 examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json create mode 100644 examples/cloud-operations/network-dashboard/main.tf create mode 100644 examples/cloud-operations/network-dashboard/tests/README.md create mode 100644 examples/cloud-operations/network-dashboard/tests/test.tf create mode 100644 examples/cloud-operations/network-dashboard/tests/variables.tf create mode 100644 examples/cloud-operations/network-dashboard/variables.tf diff --git a/examples/cloud-operations/network-dashboard/LICENSE b/examples/cloud-operations/network-dashboard/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/examples/cloud-operations/network-dashboard/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md new file mode 100644 index 00000000..5ad3e638 --- /dev/null +++ b/examples/cloud-operations/network-dashboard/README.md @@ -0,0 +1,37 @@ +# Networking Dashboard + +This repository provides an end-to-end solution to gather some GCP Networking quotas and limits (that cannot be seen in the GCP console today) and display them in a dashboard. +The goal is to allow for better visibility of these limits, facilitating capacity planning and avoiding hitting these limits. + +## Usage + +Clone this repository, then go through the following steps to create resources: +- Create a terraform.tfvars file with the following content: + - organization_id = "[YOUR-ORG-ID]" + - billing_account = "[YOUR-BILLING-ACCOUNT]" + - monitoring_project_id = "project-0" # Monitoring project where the dashboard will be created and the solution deployed + - monitored_projects_list = ["project-1", "project2"] # Projects to be monitored by the solution +- `terraform init` +- `terraform apply` + +Once the resources are deployed, go to the following page to see the dashboard: https://console.cloud.google.com/monitoring/dashboards?project=. +A dashboard called "quotas-utilization" should be created. + +The Cloud Function runs every 5 minutes by default so you should start getting some data points after a few minutes. +You can change this frequency by modifying the "schedule_cron" variable in variables.tf. + +Once done testing, you can clean up resources by running `terraform destroy`. + +## Supported limits and quotas +The Cloud Function currently tracks usage, limit and utilization of: +- active VPC peerings per VPC +- VPC peerings per VPC +- instances per VPC +- instances per VPC peering group +- Subnet IP ranges per VPC peering group +- internal forwarding rules for internal L4 load balancers per VPC +- internal forwarding rules for internal L7 load balancers per VPC +- internal forwarding rules for internal L4 load balancers per VPC peering group +- internal forwarding rules for internal L7 load balancers per VPC peering group + +It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py new file mode 100644 index 00000000..1cd603bf --- /dev/null +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -0,0 +1,588 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from google.cloud import monitoring_v3 +from googleapiclient import discovery +from google.api import metric_pb2 as ga_metric +import time +import os +import google.api_core +import re +import random + +monitored_projects_list = os.environ.get("monitored_projects_list").split(",") # list of projects from which function will get quotas information +monitoring_project_id = os.environ.get("monitoring_project_id") # project where the metrics and dahsboards will be created +monitoring_project_link = f"projects/{monitoring_project_id}" +service = discovery.build('compute', 'v1') + +# DEFAULT LIMITS +limit_vpc_peer = os.environ.get("LIMIT_VPC_PEER").split(",") # 25 +limit_l4 = os.environ.get("LIMIT_L4").split(",") # 75 +limit_l7 = os.environ.get("LIMIT_L7").split(",") # 75 +limit_instances = os.environ.get("LIMIT_INSTANCES").split(",") # ["default_value", "15000"] +limit_instances_ppg = os.environ.get("LIMIT_INSTANCES_PPG").split(",") # 15000 +limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",") # 400 +limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",") # 175 +limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",") # 175 + +# Cloud Function entry point +def quotas(request): + global client, interval + client, interval = create_client() + + # Instances per VPC + instance_metric = create_gce_instances_metrics() + get_gce_instances_data(instance_metric) + + # Number of VPC peerings per VPC + vpc_peering_active_metric, vpc_peering_metric = create_vpc_peering_metrics() + get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric) + + # Internal L4 Forwarding Rules per VPC + forwarding_rules_metric = create_l4_forwarding_rules_metric() + get_l4_forwarding_rules_data(forwarding_rules_metric) + + # Internal L4 Forwarding Rules per VPC peering group + # Existing GCP Monitoring metrics for L4 Forwarding Rules per Network + l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" + l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" + + l4_forwarding_rules_ppg_metric = create_l4_forwarding_rules_ppg_metric() + get_pgg_data(l4_forwarding_rules_ppg_metric, l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg) + + # Internal L7 Forwarding Rules per VPC peering group + # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network + l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" + l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" + + l7_forwarding_rules_ppg_metric = create_l7_forwarding_rules_ppg_metric() + get_pgg_data(l7_forwarding_rules_ppg_metric, l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg) + + # Subnet ranges per VPC peering group + # Existing GCP Monitoring metrics for Subnet Ranges per Network + subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" + subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" + + subnet_ranges_ppg_metric = create_subnet_ranges_ppg_metric() + get_pgg_data(subnet_ranges_ppg_metric, subnet_ranges_usage, subnet_ranges_limit, limit_subnets) + + # GCE Instances per VPC peering group + # Existing GCP Monitoring metrics for GCE per Network + gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" + gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" + + gce_instances_metric = create_gce_instances_ppg_metric() + get_pgg_data(gce_instances_metric, gce_instances_usage, gce_instances_limit, limit_instances_ppg) + + return 'Function executed successfully' + +def create_client(): + try: + client = monitoring_v3.MetricServiceClient() + now = time.time() + seconds = int(now) + nanos = int((now - seconds) * 10 ** 9) + interval = monitoring_v3.TimeInterval( + { + "end_time": {"seconds": seconds, "nanos": nanos}, + "start_time": {"seconds": (seconds - 86400), "nanos": nanos}, + }) + return (client, interval) + except Exception as e: + raise Exception("Error occurred creating the client: {}".format(e)) + +# Creates usage, limit and utilization Cloud monitoring metrics for GCE instances +# Returns a dictionary with the names and descriptions of the created metrics +def create_gce_instances_metrics(): + instance_metric = {} + instance_metric["usage_name"] = "number_of_instances_usage" + instance_metric["limit_name"] = "number_of_instances_limit" + instance_metric["utilization_name"] = "number_of_instances_utilization" + + instance_metric["usage_description"] = "Number of instances per VPC network - usage." + instance_metric["limit_description"] = "Number of instances per VPC network - effective limit." + instance_metric["utilization_description"] = "Number of instances per VPC network - utilization." + + create_metric(instance_metric["usage_name"], instance_metric["usage_description"]) + create_metric(instance_metric["limit_name"], instance_metric["limit_description"]) + create_metric(instance_metric["utilization_name"], instance_metric["utilization_description"]) + + return instance_metric + +# Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing +def create_metric(metric_name, description): + client = monitoring_v3.MetricServiceClient() + + metric_link = f"custom.googleapis.com/{metric_name}" + types = [] + for desc in client.list_metric_descriptors(name=monitoring_project_link): + types.append(desc.type) + + # If the metric doesn't exist yet, then we create it + if metric_link not in types: + descriptor = ga_metric.MetricDescriptor() + descriptor.type = f"custom.googleapis.com/{metric_name}" + descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE + descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE + descriptor.description = description + descriptor = client.create_metric_descriptor(name=monitoring_project_link, metric_descriptor=descriptor) + print("Created {}.".format(descriptor.name)) + +def get_gce_instances_data(instance_metric): + + # Existing GCP Monitoring metrics for GCE instances + metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" + metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" + + for project in monitored_projects_list: + network_dict = get_networks(project) + + current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) + + current_quota_usage_view = customize_quota_view(current_quota_usage) + current_quota_limit_view = customize_quota_view(current_quota_limit) + + for net in network_dict: + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_instances) + write_data_to_metric(project, net['usage'], instance_metric["usage_name"], net['network name']) + write_data_to_metric(project, net['limit'], instance_metric["limit_name"], net['network name']) + write_data_to_metric(project, net['usage']/ net['limit'], instance_metric["utilization_name"], net['network name']) + + print(f"Wrote number of instances to metric for projects/{project}") + + +# Creates 2 metrics: VPC peerings per VPC and active VPC peerings per VPC +def create_vpc_peering_metrics(): + vpc_peering_active_metric = {} + vpc_peering_active_metric["usage_name"] = "number_of_active_vpc_peerings_usage" + vpc_peering_active_metric["limit_name"] = "number_of_active_vpc_peerings_limit" + vpc_peering_active_metric["utilization_name"] = "number_of_active_vpc_peerings_utilization" + + vpc_peering_active_metric["usage_description"] = "Number of active VPC Peerings per VPC - usage." + vpc_peering_active_metric["limit_description"] = "Number of active VPC Peerings per VPC - effective limit." + vpc_peering_active_metric["utilization_description"] = "Number of active VPC Peerings per VPC - utilization." + + vpc_peering_metric = {} + vpc_peering_metric["usage_name"] = "number_of_vpc_peerings_usage" + vpc_peering_metric["limit_name"] = "number_of_vpc_peerings_limit" + vpc_peering_metric["utilization_name"] = "number_of_vpc_peerings_utilization" + + vpc_peering_metric["usage_description"] = "Number of VPC Peerings per VPC - usage." + vpc_peering_metric["limit_description"] = "Number of VPC Peerings per VPC - effective limit." + vpc_peering_metric["utilization_description"] = "Number of VPC Peerings per VPC - utilization." + + create_metric(vpc_peering_active_metric["usage_name"], vpc_peering_active_metric["usage_description"]) + create_metric(vpc_peering_active_metric["limit_name"], vpc_peering_active_metric["limit_description"]) + create_metric(vpc_peering_active_metric["utilization_name"], vpc_peering_active_metric["utilization_description"]) + + create_metric(vpc_peering_metric["usage_name"], vpc_peering_metric["usage_description"]) + create_metric(vpc_peering_metric["limit_name"], vpc_peering_metric["limit_description"]) + create_metric(vpc_peering_metric["utilization_name"], vpc_peering_metric["utilization_description"]) + + return vpc_peering_active_metric, vpc_peering_metric + +# Populates data for VPC peerings per VPC and active VPC peerings per VPC +def get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric): + for project in monitored_projects_list: + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_vpc_peer) + for peering in active_vpc_peerings: + write_data_to_metric(project, peering['active_peerings'], vpc_peering_active_metric["usage_name"], peering['network name']) + write_data_to_metric(project, peering['network_limit'], vpc_peering_active_metric["limit_name"], peering['network name']) + write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], vpc_peering_active_metric["utilization_name"], peering['network name']) + print("Wrote number of active VPC peerings to custom metric for project:", project) + + for peering in vpc_peerings: + write_data_to_metric(project, peering['peerings'], vpc_peering_metric["usage_name"], peering['network name']) + write_data_to_metric(project, peering['network_limit'], vpc_peering_metric["limit_name"], peering['network name']) + write_data_to_metric(project, peering['peerings'] / peering['network_limit'], vpc_peering_metric["utilization_name"], peering['network name']) + print("Wrote number of VPC peerings to custom metric for project:", project) + +# gathers number of VPC peerings, active VPC peerings and limits, returns 2 dictionaries: active_vpc_peerings and vpc_peerings (including inactive and active ones) +def gather_vpc_peerings_data(project_id, limit_list): + active_peerings_dict = [] + peerings_dict = [] + request = service.networks().list(project=project_id) + response = request.execute() + if 'items' in response: + for network in response['items']: + if 'peerings' in network: + STATE = network['peerings'][0]['state'] + if STATE == "ACTIVE": + active_peerings_count = len(network['peerings']) + else: + active_peerings_count = 0 + + peerings_count = len(network['peerings']) + else: + peerings_count = 0 + active_peerings_count = 0 + + active_d = {'project_id': project_id,'network name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)} + active_peerings_dict.append(active_d) + d = {'project_id': project_id,'network name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)} + peerings_dict.append(d) + + return active_peerings_dict, peerings_dict + +# Check if the VPC has a specific limit for a specific metric, if so, returns that limit, if not, returns the default limit +def get_limit(network_name, limit_list): + if network_name in limit_list: + return int(limit_list[limit_list.index(network_name) + 1]) + else: + if 'default_value' in limit_list: + return int(limit_list[limit_list.index('default_value') + 1]) + else: + return 0 + +# Creates the custom metrics for L4 internal forwarding Rules +def create_l4_forwarding_rules_metric(): + forwarding_rules_metric = {} + forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_usage" + forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_limit" + forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_utilization" + + forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - usage." + forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - effective limit." + forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization." + + create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) + create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) + create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) + + return forwarding_rules_metric + +def get_l4_forwarding_rules_data(forwarding_rules_metric): + + # Existing GCP Monitoring metrics for L4 Forwarding Rules + l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" + l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" + + for project in monitored_projects_list: + network_dict = get_networks(project) + + current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) + + current_quota_usage_view = customize_quota_view(current_quota_usage) + current_quota_limit_view = customize_quota_view(current_quota_limit) + + for net in network_dict: + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_l4) + write_data_to_metric(project, net['usage'], forwarding_rules_metric["usage_name"], net['network name']) + write_data_to_metric(project, net['limit'], forwarding_rules_metric["limit_name"], net['network name']) + write_data_to_metric(project, net['usage']/ net['limit'], forwarding_rules_metric["utilization_name"], net['network name']) + + print(f"Wrote number of L4 forwarding rules to metric for projects/{project}") + +# Creates the custom metrics for L4 internal forwarding Rules per VPC Peering Group +def create_l4_forwarding_rules_ppg_metric(): + forwarding_rules_metric = {} + forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_ppg_usage" + forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_ppg_limit" + forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_ppg_utilization" + + forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - usage." + forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - effective limit." + forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - utilization." + + create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) + create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) + create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) + + return forwarding_rules_metric + +# Creates the custom metrics for L7 internal forwarding Rules per VPC Peering Group +def create_l7_forwarding_rules_ppg_metric(): + forwarding_rules_metric = {} + forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l7_ppg_usage" + forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l7_ppg_limit" + forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l7_ppg_utilization" + + forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - usage." + forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - effective limit." + forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - utilization." + + create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) + create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) + create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) + + return forwarding_rules_metric + +def create_subnet_ranges_ppg_metric(): + metric = {} + + metric["usage_name"] = "number_of_subnet_IP_ranges_usage" + metric["limit_name"] = "number_of_subnet_IP_ranges_effective_limit" + metric["utilization_name"] = "number_of_subnet_IP_ranges_utilization" + + metric["usage_description"] = "Number of Subnet Ranges per peering group - usage." + metric["limit_description"] = "Number of Subnet Ranges per peering group - effective limit." + metric["utilization_description"] = "Number of Subnet Ranges per peering group - utilization." + + create_metric(metric["usage_name"], metric["usage_description"]) + create_metric(metric["limit_name"], metric["limit_description"]) + create_metric(metric["utilization_name"], metric["utilization_description"]) + + return metric + +def create_gce_instances_ppg_metric(): + metric = {} + + metric["usage_name"] = "number_of_instances_ppg_usage" + metric["limit_name"] = "number_of_instances_ppg_limit" + metric["utilization_name"] = "number_of_instances_ppg_utilization" + + metric["usage_description"] = "Number of instances per peering group - usage." + metric["limit_description"] = "Number of instances per peering group - effective limit." + metric["utilization_description"] = "Number of instances per peering group - utilization." + + create_metric(metric["usage_name"], metric["usage_description"]) + create_metric(metric["limit_name"], metric["limit_description"]) + create_metric(metric["utilization_name"], metric["utilization_description"]) + + return metric + +# Populates data for the custom metrics for L4 internal forwarding Rules per Peering Group +def get_pgg_data(forwarding_rules_ppg_metric, usage_metric, limit_metric, limit_ppg): + + for project in monitored_projects_list: + network_dict_list = gather_peering_data(project) + # Network dict list is a list of dictionary (one for each network) + # For each network, this dictionary contains: + # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) + # peerings is a list of dictionary (one for each peered network) and contains: + # project_id, network_name, network_id + + # For each network in this GCP project + for network_dict in network_dict_list: + current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric) + current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) + + current_quota_usage_view = customize_quota_view(current_quota_usage) + current_quota_limit_view = customize_quota_view(current_quota_limit) + + usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg) + # Here we add usage and limit to the network dictionary + network_dict["usage"] = usage + network_dict["limit"] = limit + + # For every peered network, get usage and limits + for peered_network in network_dict['peerings']: + peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric)) + peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) + + usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg) + # Here we add usage and limit to the peered network dictionary + peered_network["usage"] = usage + peered_network["limit"] = limit + + count_effective_limit(project, network_dict, forwarding_rules_ppg_metric["usage_name"], forwarding_rules_ppg_metric["limit_name"], forwarding_rules_ppg_metric["utilization_name"], limit_ppg) + print(f"Wrote {forwarding_rules_ppg_metric['usage_name']} to metric for peering group {network_dict['network_name']} in {project}") + +# Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to metric +# https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit +def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg): + + if network_dict['peerings'] == []: + return + + # Get usage: Sums usage for current network + all peered networks + peering_group_usage = network_dict['usage'] + for peered_network in network_dict['peerings']: + peering_group_usage += peered_network['usage'] + + # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) + limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg)) + + # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network + limit_step2 = [] + for peered_network in network_dict['peerings']: + limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg))) + + # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 + limit_step3 = min(limit_step2) + + # Calculates effective limit: Step 4: Find maximum from step 1 and step 3 + effective_limit = max(limit_step1, limit_step3) + utilization = peering_group_usage / effective_limit + + write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name']) + write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name']) + write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) + +# Takes a project id as argument (and service for the GCP API call) and return a dictionary with the list of networks +def get_networks(project_id): + request = service.networks().list(project=project_id) + response = request.execute() + network_dict = [] + if 'items' in response: + for network in response['items']: + NETWORK = network['name'] + ID = network['id'] + d = {'project_id':project_id,'network name':NETWORK,'network id':ID} + network_dict.append(d) + return network_dict + +# gathers data for peered networks for the given project_id +def gather_peering_data(project_id): + request = service.networks().list(project=project_id) + response = request.execute() + + # list of networks in that project + network_list = [] + if 'items' in response: + for network in response['items']: + net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]} + if 'peerings' in network: + STATE = network['peerings'][0]['state'] + if STATE == "ACTIVE": + for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}" + start = peered_network['network'].find("projects/") + len('projects/') + end = peered_network['network'].find("/global") + peered_project = peered_network['network'][start:end] + peered_network_name = peered_network['network'].split("networks/")[1] + peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)} + net["peerings"].append(peered_net) + network_list.append(net) + return network_list + +def get_network_id(project_id, network_name): + request = service.networks().list(project=project_id) + response = request.execute() + + network_id = 0 + + if 'items' in response: + for network in response['items']: + if network['name'] == network_name: + network_id = network['id'] + break + + if network_id == 0: + print(f"Error: network_id not found for {network_name} in {project_id}") + + return network_id + +# retrieves quota for "type" in project_link, otherwise returns null (assume 0 for building comparison vs limits) +def get_quota_current_usage(project_link, type): + results = client.list_time_series(request={ + "name": project_link, + "filter": f'metric.type = "{type}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) + results_list = list(results) + return (results_list) + +# retrieves quota for services limits +def get_quota_current_limit(project_link, type): + results = client.list_time_series(request={ + "name": project_link, + "filter": f'metric.type = "{type}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) + results_list = list(results) + return (results_list) + +# Customize the quota output +def customize_quota_view(quota_results): + quotaViewList = [] + for result in quota_results: + quotaViewJson = {} + quotaViewJson.update(dict(result.resource.labels)) + quotaViewJson.update(dict(result.metric.labels)) + for val in result.points: + quotaViewJson.update({'value': val.value.int64_value}) + quotaViewList.append(quotaViewJson) + return (quotaViewList) + +# Takes a network dictionary and updates it with the quota usage and limits values +def set_usage_limits(network, quota_usage, quota_limit, limit_list): + if quota_usage: + for net in quota_usage: + if net['network_id'] == network['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network['usage'] = net['value'] # set network usage in dictionary + break + else: + network['usage'] = 0 # if network does not appear in GCP quotas + else: + network['usage'] = 0 # if quotas does not appear in GCP quotas + + if quota_limit: + for net in quota_limit: + if net['network_id'] == network['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network['limit'] = net['value'] # set network limit in dictionary + break + else: + if network['network name'] in limit_list: # if network limit is in the environmental variables + network['limit'] = int(limit_list[limit_list.index(network['network name']) + 1]) + else: + network['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value + else: # if quotas does not appear in GCP quotas + if network['network name'] in limit_list: + network['limit'] = int(limit_list[limit_list.index(network['network name']) + 1]) # ["default", 100, "networkname", 200] + else: + network['limit'] = int(limit_list[limit_list.index('default_value') + 1]) + +# Takes a network dictionary (with at least network_id and network_name) and returns usage and limit for that network +def get_usage_limit(network, quota_usage, quota_limit, limit_list): + usage = 0 + limit = 0 + + if quota_usage: + for net in quota_usage: + if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + usage = net['value'] # set network usage in dictionary + break + + if quota_limit: + for net in quota_limit: + if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + limit = net['value'] # set network limit in dictionary + break + else: + if network['network_name'] in limit_list: # if network limit is in the environmental variables + limit = int(limit_list[limit_list.index(network['network_name']) + 1]) + else: + limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value + else: # if quotas does not appear in GCP quotas + if network['network_name'] in limit_list: + limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200] + else: + limit = int(limit_list[limit_list.index('default_value') + 1]) + + return usage, limit + +# Writes data to Cloud Monitoring data +# Note that the monitoring_project_id here should be the monitoring project where the metrics are written +# and monitored_project_id is the monitored project, containing the network and resources +def write_data_to_metric(monitored_project_id, value, metric_name, network_name): + series = monitoring_v3.TimeSeries() + series.metric.type = f"custom.googleapis.com/{metric_name}" + series.resource.type = "global" + series.metric.labels["network_name"] = network_name + series.metric.labels["project"] = monitored_project_id + + now = time.time() + seconds = int(now) + nanos = int((now - seconds) * 10 ** 9) + interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}}) + point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) + series.points = [point] + + client.create_time_series(name=monitoring_project_link, time_series=[series]) \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt new file mode 100644 index 00000000..5e91a8d8 --- /dev/null +++ b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt @@ -0,0 +1,8 @@ +regex +google-api-python-client +google-auth +google-auth-httplib2 +google-cloud-logging +google-cloud-monitoring +oauth2client +google-api-core \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json new file mode 100644 index 00000000..af812061 --- /dev/null +++ b/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -0,0 +1,351 @@ +{ + "displayName": "quotas_utilization", + "mosaicLayout": { + "columns": 12, + "tiles": [ + { + "height": 4, + "widget": { + "title": "internal_forwarding_rules_l4_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "1800s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6 + }, + { + "height": 4, + "widget": { + "title": "internal_forwarding_rules_l7_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6 + }, + { + "height": 4, + "widget": { + "title": "number_of_instances_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "yPos": 8 + }, + { + "height": 4, + "widget": { + "title": "number_of_vpc_peerings_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, + "yPos": 4 + }, + { + "height": 4, + "widget": { + "title": "number_of_active_vpc_peerings_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_INTERPOLATE" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "yPos": 4 + }, + { + "height": 4, + "widget": { + "title": "number_of_subnet_IP_ranges_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, + "yPos": 8 + }, + { + "height": 4, + "widget": { + "title": "internal_forwarding_rules_l4_ppg_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "yPos": 12 + }, + { + "height": 4, + "widget": { + "title": "internal_forwarding_rules_l7_ppg_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, + "yPos": 12 + }, + { + "height": 4, + "widget": { + "title": "number_of_instances_ppg_utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "yPos": 16 + } + ] + } +} \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf new file mode 100644 index 00000000..08dabef4 --- /dev/null +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + project_id_list = toset(var.monitored_projects_list) + projects = join(",", local.project_id_list) + + limit_subnets_list = tolist(var.limit_subnets) + limit_subnets = join(",", local.limit_subnets_list) + limit_instances_list = tolist(var.limit_instances) + limit_instances = join(",", local.limit_instances_list) + limit_instances_ppg_list = tolist(var.limit_instances_ppg) + limit_instances_ppg = join(",", local.limit_instances_ppg_list) + limit_vpc_peer_list = tolist(var.limit_vpc_peer) + limit_vpc_peer = join(",", local.limit_vpc_peer_list) + limit_l4_list = tolist(var.limit_l4) + limit_l4 = join(",", local.limit_l4_list) + limit_l7_list = tolist(var.limit_l7) + limit_l7 = join(",", local.limit_l7_list) + limit_l4_ppg_list = tolist(var.limit_l4_ppg) + limit_l4_ppg = join(",", local.limit_l4_ppg_list) + limit_l7_ppg_list = tolist(var.limit_l7_ppg) + limit_l7_ppg = join(",", local.limit_l7_ppg_list) +} + +################################################ +# Monitoring project creation # +################################################ + +module "project-monitoring" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + name = "monitoring" + parent = "organizations/${var.organization_id}" + prefix = var.prefix + billing_account = var.billing_account + services = var.project_monitoring_services +} + +################################################ +# Service account creation and IAM permissions # +################################################ + +module "service-account-function" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/iam-service-account" + project_id = module.project-monitoring.project_id + name = "sa-dash" + generate_key = false + + # Required IAM permissions for this service account are: + # 1) compute.networkViewer on projects to be monitored (I gave it at organization level for now for simplicity) + # 2) monitoring viewer on the projects to be monitored (I gave it at organization level for now for simplicity) + iam_organization_roles = { + "${var.organization_id}" = [ + "roles/compute.networkViewer", + "roles/monitoring.viewer", + ] + } + + iam_project_roles = { + "${module.project-monitoring.project_id}" = [ + "roles/monitoring.metricWriter" + ] + } +} + +################################################ +# Cloud Function configuration (& Scheduler) # +################################################ + +# Create an app engine application (required for Cloud Scheduler) +resource "google_app_engine_application" "scheduler_app" { + project = module.project-monitoring.project_id + # "europe-west1" is called "europe-west" and "us-central1" is "us-central" for App Engine, see https://cloud.google.com/appengine/docs/locations + location_id = var.region == "europe-west1" || var.region == "us-central1" ? substr(var.region, 0, length(var.region) - 1) : var.region +} + +# Create a storage bucket for the Cloud Function's code +resource "google_storage_bucket" "bucket" { + name = "net-quotas-bucket" + location = "EU" + project = module.project-monitoring.project_id + +} + +data "archive_file" "file" { + type = "zip" + source_dir = "cloud-function" + output_path = "cloud-function.zip" + depends_on = [google_storage_bucket.bucket] +} + +resource "google_storage_bucket_object" "archive" { + # md5 hash in the bucket object name to redeploy the Cloud Function when the code is modified + name = format("cloud-function#%s", data.archive_file.file.output_md5) + bucket = google_storage_bucket.bucket.name + source = "cloud-function.zip" + depends_on = [data.archive_file.file] +} + +resource "google_cloudfunctions_function" "function_quotas" { + name = "function-quotas" + project = module.project-monitoring.project_id + region = var.region + description = "Function which creates metric to show number, limit and utlizitation." + runtime = "python39" + + available_memory_mb = 512 + source_archive_bucket = google_storage_bucket.bucket.name + source_archive_object = google_storage_bucket_object.archive.name + service_account_email = module.service-account-function.email + + timeout = 180 + entry_point = "quotas" + trigger_http = true + + + environment_variables = { + monitored_projects_list = local.projects + monitoring_project_id = module.project-monitoring.project_id + + LIMIT_SUBNETS = local.limit_subnets + LIMIT_INSTANCES = local.limit_instances + LIMIT_INSTANCES_PPG = local.limit_instances_ppg + LIMIT_VPC_PEER = local.limit_vpc_peer + LIMIT_L4 = local.limit_l4 + LIMIT_L7 = local.limit_l7 + LIMIT_L4_PPG = local.limit_l4_ppg + LIMIT_L7_PPG = local.limit_l7_ppg + } +} + +resource "google_cloud_scheduler_job" "job" { + name = "scheduler-net-dash" + project = module.project-monitoring.project_id + region = var.region + description = "Cloud Scheduler job to trigger the Networking Dashboard Cloud Function" + schedule = var.schedule_cron + + retry_config { + retry_count = 1 + } + + http_target { + http_method = "POST" + uri = google_cloudfunctions_function.function_quotas.https_trigger_url + # We could pass useful data in the body later + body = base64encode("{\"foo\":\"bar\"}") + } +} + +# TODO: How to secure Cloud Function invokation? Not member = "allUsers" but specific Scheduler service Account? +# Maybe "service-YOUR_PROJECT_NUMBER@gcp-sa-cloudscheduler.iam.gserviceaccount.com"? + +resource "google_cloudfunctions_function_iam_member" "invoker" { + project = module.project-monitoring.project_id + region = var.region + cloud_function = google_cloudfunctions_function.function_quotas.name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" +} + +################################################ +# Cloud Monitoring Dashboard creation # +################################################ + +resource "google_monitoring_dashboard" "dashboard" { + dashboard_json = file("${path.module}/dashboards/quotas-utilization.json") + project = module.project-monitoring.project_id +} diff --git a/examples/cloud-operations/network-dashboard/tests/README.md b/examples/cloud-operations/network-dashboard/tests/README.md new file mode 100644 index 00000000..6e4779d4 --- /dev/null +++ b/examples/cloud-operations/network-dashboard/tests/README.md @@ -0,0 +1 @@ +Creating here resources to test the Cloud Function and ensuring metrics are correctly populated \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/tests/test.tf b/examples/cloud-operations/network-dashboard/tests/test.tf new file mode 100644 index 00000000..9b381648 --- /dev/null +++ b/examples/cloud-operations/network-dashboard/tests/test.tf @@ -0,0 +1,279 @@ +# Creating test infrastructure + +resource "google_folder" "test-net-dash" { + display_name = "test-net-dash" + parent = "organizations/${var.organization_id}" +} + +##### Creating host projects, VPCs, service projects ##### + +module "project-hub" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + name = "test-host-hub" + parent = google_folder.test-net-dash.name + prefix = var.prefix + billing_account = var.billing_account + services = var.project_vm_services + + shared_vpc_host_config = { + enabled = true + service_projects = [] # defined later + } +} + +module "vpc-hub" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc" + project_id = module.project-hub.project_id + name = "vpc-hub" + subnets = [ + { + ip_cidr_range = "10.0.10.0/24" + name = "subnet-hub-1" + region = var.region + secondary_ip_range = {} + } + ] +} + +module "project-svc-hub" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + parent = google_folder.test-net-dash.name + billing_account = var.billing_account + prefix = var.prefix + name = "test-svc-hub" + services = var.project_vm_services + + shared_vpc_service_config = { + attach = true + host_project = module.project-hub.project_id + } +} + +module "project-prod" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + name = "test-host-prod" + parent = google_folder.test-net-dash.name + prefix = var.prefix + billing_account = var.billing_account + services = var.project_vm_services + + shared_vpc_host_config = { + enabled = true + service_projects = [] # defined later + } +} + +module "vpc-prod" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc" + project_id = module.project-prod.project_id + name = "vpc-prod" + subnets = [ + { + ip_cidr_range = "10.0.20.0/24" + name = "subnet-prod-1" + region = var.region + secondary_ip_range = {} + } + ] +} + +module "project-svc-prod" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + parent = google_folder.test-net-dash.name + billing_account = var.billing_account + prefix = var.prefix + name = "test-svc-prod" + services = var.project_vm_services + + shared_vpc_service_config = { + attach = true + host_project = module.project-prod.project_id + } +} + +module "project-dev" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + name = "test-host-dev" + parent = google_folder.test-net-dash.name + prefix = var.prefix + billing_account = var.billing_account + services = var.project_vm_services + + shared_vpc_host_config = { + enabled = true + service_projects = [] # defined later + } +} + +module "vpc-dev" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc" + project_id = module.project-dev.project_id + name = "vpc-dev" + subnets = [ + { + ip_cidr_range = "10.0.30.0/24" + name = "subnet-dev-1" + region = var.region + secondary_ip_range = {} + } + ] +} + +module "project-svc-dev" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + parent = google_folder.test-net-dash.name + billing_account = var.billing_account + prefix = var.prefix + name = "test-svc-dev" + services = var.project_vm_services + + shared_vpc_service_config = { + attach = true + host_project = module.project-dev.project_id + } +} + +##### Creating VPC peerings ##### + +module "hub-to-prod-peering" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + local_network = module.vpc-hub.self_link + peer_network = module.vpc-prod.self_link +} + +module "prod-to-hub-peering" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + local_network = module.vpc-prod.self_link + peer_network = module.vpc-hub.self_link + depends_on = [module.hub-to-prod-peering] +} + +module "hub-to-dev-peering" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + local_network = module.vpc-hub.self_link + peer_network = module.vpc-dev.self_link +} + +module "dev-to-hub-peering" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + local_network = module.vpc-dev.self_link + peer_network = module.vpc-hub.self_link + depends_on = [module.hub-to-dev-peering] +} + +##### Creating VMs ##### + +resource "google_compute_instance" "test-vm-prod1" { + project = module.project-svc-prod.project_id + name = "test-vm-prod1" + machine_type = "f1-micro" + zone = var.zone + + tags = ["${var.region}"] + + boot_disk { + initialize_params { + image = "debian-cloud/debian-9" + } + } + + network_interface { + subnetwork = module.vpc-prod.subnet_self_links["${var.region}/subnet-prod-1"] + subnetwork_project = module.project-prod.project_id + } + + allow_stopping_for_update = true +} + +resource "google_compute_instance" "test-vm-prod2" { + project = module.project-prod.project_id + name = "test-vm-prod2" + machine_type = "f1-micro" + zone = var.zone + + tags = ["${var.region}"] + + boot_disk { + initialize_params { + image = "debian-cloud/debian-9" + } + } + + network_interface { + subnetwork = module.vpc-prod.subnet_self_links["${var.region}/subnet-prod-1"] + subnetwork_project = module.project-prod.project_id + } + + allow_stopping_for_update = true +} + +resource "google_compute_instance" "test-vm-dev1" { + count = 10 + project = module.project-svc-dev.project_id + name = "test-vm-dev${count.index}" + machine_type = "f1-micro" + zone = var.zone + + tags = ["${var.region}"] + + boot_disk { + initialize_params { + image = "debian-cloud/debian-9" + } + } + + network_interface { + subnetwork = module.vpc-dev.subnet_self_links["${var.region}/subnet-dev-1"] + subnetwork_project = module.project-dev.project_id + } + + allow_stopping_for_update = true +} + +resource "google_compute_instance" "test-vm-hub1" { + project = module.project-svc-hub.project_id + name = "test-vm-hub1" + machine_type = "f1-micro" + zone = var.zone + + tags = ["${var.region}"] + + boot_disk { + initialize_params { + image = "debian-cloud/debian-9" + } + } + + network_interface { + subnetwork = module.vpc-hub.subnet_self_links["${var.region}/subnet-hub-1"] + subnetwork_project = module.project-hub.project_id + } + + allow_stopping_for_update = true +} + +# Forwarding Rules + +resource "google_compute_forwarding_rule" "forwarding-rule-dev" { + name = "forwarding-rule-dev" + project = module.project-svc-dev.project_id + network = module.vpc-dev.self_link + subnetwork = module.vpc-dev.subnet_self_links["${var.region}/subnet-dev-1"] + + region = var.region + backend_service = google_compute_region_backend_service.test-backend.id + ip_protocol = "TCP" + load_balancing_scheme = "INTERNAL" + all_ports = true + allow_global_access = true + +} + +# backend service +resource "google_compute_region_backend_service" "test-backend" { + name = "test-backend" + region = var.region + project = module.project-svc-dev.project_id + protocol = "TCP" + load_balancing_scheme = "INTERNAL" +} diff --git a/examples/cloud-operations/network-dashboard/tests/variables.tf b/examples/cloud-operations/network-dashboard/tests/variables.tf new file mode 100644 index 00000000..c1833b7e --- /dev/null +++ b/examples/cloud-operations/network-dashboard/tests/variables.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "organization_id" { + description = "The organization id for the associated services" +} + +variable "billing_account" { + description = "The ID of the billing account to associate this project with" +} + +variable "prefix" { + description = "Customer name to use as prefix for resources' naming" + default = "net-dash" +} + +variable "project_vm_services" { + description = "Service APIs enabled by default in new projects." + default = [ + "cloudbilling.googleapis.com", + "compute.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "servicenetworking.googleapis.com", + ] +} + +variable "region" { + description = "Region used to deploy subnets" + default = "europe-west1" +} + +variable "zone" { + description = "Zone used to deploy vms" + default = "europe-west1-b" +} \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/variables.tf b/examples/cloud-operations/network-dashboard/variables.tf new file mode 100644 index 00000000..b3b0c23e --- /dev/null +++ b/examples/cloud-operations/network-dashboard/variables.tf @@ -0,0 +1,150 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "organization_id" { + description = "The organization id for the associated services" +} + +variable "billing_account" { + description = "The ID of the billing account to associate this project with" +} + +variable "prefix" { + description = "Customer name to use as prefix for resources' naming" + default = "net-dash" +} + +# Not used for now as I am creating the monitoring project in my main.tf file +variable "monitoring_project_id" { + type = string + description = "ID of the monitoring project, where the Cloud Function and dashboards will be deployed." +} + +# TODO: support folder instead of a list of projects? +variable "monitored_projects_list" { + type = list(string) + description = "ID of the projects to be monitored (where limits and quotas data will be pulled)" +} + +variable "schedule_cron" { + description = "Cron format schedule to run the Cloud Function. Default is every 5 minutes." + default = "*/5 * * * *" +} + +variable "project_monitoring_services" { + description = "Service APIs enabled by default in new projects." + default = [ + "cloudbilling.googleapis.com", + "cloudbuild.googleapis.com", + "cloudresourcemanager.googleapis.com", + "cloudscheduler.googleapis.com", + "compute.googleapis.com", + "cloudfunctions.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "oslogin.googleapis.com", + "servicenetworking.googleapis.com", + "serviceusage.googleapis.com", + ] +} + +variable "project_vm_services" { + description = "Service APIs enabled by default in new projects." + default = [ + "cloudbilling.googleapis.com", + "compute.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "servicenetworking.googleapis.com", + ] +} + +variable "region" { + description = "Region used to deploy subnets" + default = "europe-west1" +} + +variable "zone" { + description = "Zone used to deploy vms" + default = "europe-west1-b" +} + +variable "limit_l4" { + description = "Maximum number of forwarding rules for Internal TCP/UDP Load Balancing per network." + type = list(string) + default = [ + "default_value", "75", + ] +} + +variable "limit_l7" { + description = "Maximum number of forwarding rules for Internal HTTP(S) Load Balancing per network." + type = list(string) + default = [ + "default_value", "75", + ] +} + +variable "limit_subnets" { + description = "Maximum number of subnet IP ranges (primary and secondary) per peering group" + type = list(string) + default = [ + "default_value", "400", + ] +} + +variable "limit_instances" { + description = "Maximum number of instances per network" + type = list(string) + default = [ + "default_value", "15000", + ] +} + +variable "limit_instances_ppg" { + description = "Maximum number of instances per peering group." + type = list(string) + default = [ + "default_value", "15000", + ] +} + +variable "limit_vpc_peer" { + description = "Maximum number of peering VPC peerings per network." + type = list(string) + default = [ + "default_value", "25", + "test-vpc", "40", + ] +} + +variable "limit_l4_ppg" { + description = "Maximum number of forwarding rules for Internal TCP/UDP Load Balancing per network." + type = list(string) + default = [ + "default_value", "175", + ] +} + +variable "limit_l7_ppg" { + description = "Maximum number of forwarding rules for Internal HTTP(S) Load Balancing per network." + type = list(string) + default = [ + "default_value", "175", + ] +} \ No newline at end of file From 221557d066f071d7cb853ef11300aee6615785e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Wed, 9 Mar 2022 19:02:59 +0100 Subject: [PATCH 02/23] Pinning version for libs and terraform modules, adding docstrings, improving documentation of the code, tabs 2 spaces. --- .../network-dashboard/cloud-function/main.py | 1277 ++++++++++------- .../cloud-function/requirements.txt | 16 +- .../network-dashboard/main.tf | 4 +- .../network-dashboard/tests/test.tf | 42 +- 4 files changed, 778 insertions(+), 561 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 1cd603bf..407e99a1 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -5,7 +5,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -23,566 +23,769 @@ import google.api_core import re import random +# monitored_projects_list = os.environ.get("monitored_projects_list").split(",") # list of projects from which function will get quotas information monitoring_project_id = os.environ.get("monitoring_project_id") # project where the metrics and dahsboards will be created monitoring_project_link = f"projects/{monitoring_project_id}" service = discovery.build('compute', 'v1') -# DEFAULT LIMITS -limit_vpc_peer = os.environ.get("LIMIT_VPC_PEER").split(",") # 25 -limit_l4 = os.environ.get("LIMIT_L4").split(",") # 75 -limit_l7 = os.environ.get("LIMIT_L7").split(",") # 75 -limit_instances = os.environ.get("LIMIT_INSTANCES").split(",") # ["default_value", "15000"] -limit_instances_ppg = os.environ.get("LIMIT_INSTANCES_PPG").split(",") # 15000 -limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",") # 400 -limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",") # 175 -limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",") # 175 +# DEFAULT LIMITS: +limit_vpc_peer = os.environ.get("LIMIT_VPC_PEER").split(",") +limit_l4 = os.environ.get("LIMIT_L4").split(",") +limit_l7 = os.environ.get("LIMIT_L7").split(",") +limit_instances = os.environ.get("LIMIT_INSTANCES").split(",") +limit_instances_ppg = os.environ.get("LIMIT_INSTANCES_PPG").split(",") +limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",") +limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",") +limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",") -# Cloud Function entry point def quotas(request): - global client, interval - client, interval = create_client() + ''' + Cloud Function Entry point, called by the scheduler. - # Instances per VPC - instance_metric = create_gce_instances_metrics() - get_gce_instances_data(instance_metric) + Parameters: + request: for now, the Cloud Function is triggered by an HTTP trigger and this request correspond to the HTTP triggering request. + Returns: + 'Function executed successfully' + ''' + global client, interval + client, interval = create_client() - # Number of VPC peerings per VPC - vpc_peering_active_metric, vpc_peering_metric = create_vpc_peering_metrics() - get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric) + instance_metric = create_gce_instances_metrics() + get_gce_instances_data(instance_metric) - # Internal L4 Forwarding Rules per VPC - forwarding_rules_metric = create_l4_forwarding_rules_metric() - get_l4_forwarding_rules_data(forwarding_rules_metric) + vpc_peering_active_metric, vpc_peering_metric = create_vpc_peering_metrics() + get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric) - # Internal L4 Forwarding Rules per VPC peering group - # Existing GCP Monitoring metrics for L4 Forwarding Rules per Network - l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" - l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" + forwarding_rules_metric = create_l4_forwarding_rules_metric() + get_l4_forwarding_rules_data(forwarding_rules_metric) - l4_forwarding_rules_ppg_metric = create_l4_forwarding_rules_ppg_metric() - get_pgg_data(l4_forwarding_rules_ppg_metric, l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg) + # Existing GCP Monitoring metrics for L4 Forwarding Rules per Network + l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" + l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" - # Internal L7 Forwarding Rules per VPC peering group - # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network - l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" - l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" + l4_forwarding_rules_ppg_metric = create_l4_forwarding_rules_ppg_metric() + get_pgg_data(l4_forwarding_rules_ppg_metric, l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg) - l7_forwarding_rules_ppg_metric = create_l7_forwarding_rules_ppg_metric() - get_pgg_data(l7_forwarding_rules_ppg_metric, l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg) + # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network + l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" + l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" - # Subnet ranges per VPC peering group - # Existing GCP Monitoring metrics for Subnet Ranges per Network - subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" - subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" + l7_forwarding_rules_ppg_metric = create_l7_forwarding_rules_ppg_metric() + get_pgg_data(l7_forwarding_rules_ppg_metric, l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg) - subnet_ranges_ppg_metric = create_subnet_ranges_ppg_metric() - get_pgg_data(subnet_ranges_ppg_metric, subnet_ranges_usage, subnet_ranges_limit, limit_subnets) + # Existing GCP Monitoring metrics for Subnet Ranges per Network + subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" + subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" - # GCE Instances per VPC peering group - # Existing GCP Monitoring metrics for GCE per Network - gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" - gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" + subnet_ranges_ppg_metric = create_subnet_ranges_ppg_metric() + get_pgg_data(subnet_ranges_ppg_metric, subnet_ranges_usage, subnet_ranges_limit, limit_subnets) - gce_instances_metric = create_gce_instances_ppg_metric() - get_pgg_data(gce_instances_metric, gce_instances_usage, gce_instances_limit, limit_instances_ppg) + # Existing GCP Monitoring metrics for GCE per Network + gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" + gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - return 'Function executed successfully' + gce_instances_metric = create_gce_instances_ppg_metric() + get_pgg_data(gce_instances_metric, gce_instances_usage, gce_instances_limit, limit_instances_ppg) + + return 'Function executed successfully' def create_client(): - try: - client = monitoring_v3.MetricServiceClient() - now = time.time() - seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval( - { - "end_time": {"seconds": seconds, "nanos": nanos}, - "start_time": {"seconds": (seconds - 86400), "nanos": nanos}, - }) - return (client, interval) - except Exception as e: - raise Exception("Error occurred creating the client: {}".format(e)) + ''' + Creates the monitoring API client, that will be used to create, read and update custom metrics. -# Creates usage, limit and utilization Cloud monitoring metrics for GCE instances -# Returns a dictionary with the names and descriptions of the created metrics -def create_gce_instances_metrics(): - instance_metric = {} - instance_metric["usage_name"] = "number_of_instances_usage" - instance_metric["limit_name"] = "number_of_instances_limit" - instance_metric["utilization_name"] = "number_of_instances_utilization" - - instance_metric["usage_description"] = "Number of instances per VPC network - usage." - instance_metric["limit_description"] = "Number of instances per VPC network - effective limit." - instance_metric["utilization_description"] = "Number of instances per VPC network - utilization." - - create_metric(instance_metric["usage_name"], instance_metric["usage_description"]) - create_metric(instance_metric["limit_name"], instance_metric["limit_description"]) - create_metric(instance_metric["utilization_name"], instance_metric["utilization_description"]) - - return instance_metric - -# Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing -def create_metric(metric_name, description): + Parameters: + None + Returns: + client (monitoring_v3.MetricServiceClient): Monitoring API client + interval (monitoring_v3.TimeInterval): Interval for the metric data points (24 hours) + ''' + try: client = monitoring_v3.MetricServiceClient() - - metric_link = f"custom.googleapis.com/{metric_name}" - types = [] - for desc in client.list_metric_descriptors(name=monitoring_project_link): - types.append(desc.type) - - # If the metric doesn't exist yet, then we create it - if metric_link not in types: - descriptor = ga_metric.MetricDescriptor() - descriptor.type = f"custom.googleapis.com/{metric_name}" - descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE - descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE - descriptor.description = description - descriptor = client.create_metric_descriptor(name=monitoring_project_link, metric_descriptor=descriptor) - print("Created {}.".format(descriptor.name)) - -def get_gce_instances_data(instance_metric): - - # Existing GCP Monitoring metrics for GCE instances - metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" - metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - - for project in monitored_projects_list: - network_dict = get_networks(project) - - current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) - current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) - - current_quota_usage_view = customize_quota_view(current_quota_usage) - current_quota_limit_view = customize_quota_view(current_quota_limit) - - for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_instances) - write_data_to_metric(project, net['usage'], instance_metric["usage_name"], net['network name']) - write_data_to_metric(project, net['limit'], instance_metric["limit_name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], instance_metric["utilization_name"], net['network name']) - - print(f"Wrote number of instances to metric for projects/{project}") - - -# Creates 2 metrics: VPC peerings per VPC and active VPC peerings per VPC -def create_vpc_peering_metrics(): - vpc_peering_active_metric = {} - vpc_peering_active_metric["usage_name"] = "number_of_active_vpc_peerings_usage" - vpc_peering_active_metric["limit_name"] = "number_of_active_vpc_peerings_limit" - vpc_peering_active_metric["utilization_name"] = "number_of_active_vpc_peerings_utilization" - - vpc_peering_active_metric["usage_description"] = "Number of active VPC Peerings per VPC - usage." - vpc_peering_active_metric["limit_description"] = "Number of active VPC Peerings per VPC - effective limit." - vpc_peering_active_metric["utilization_description"] = "Number of active VPC Peerings per VPC - utilization." - - vpc_peering_metric = {} - vpc_peering_metric["usage_name"] = "number_of_vpc_peerings_usage" - vpc_peering_metric["limit_name"] = "number_of_vpc_peerings_limit" - vpc_peering_metric["utilization_name"] = "number_of_vpc_peerings_utilization" - - vpc_peering_metric["usage_description"] = "Number of VPC Peerings per VPC - usage." - vpc_peering_metric["limit_description"] = "Number of VPC Peerings per VPC - effective limit." - vpc_peering_metric["utilization_description"] = "Number of VPC Peerings per VPC - utilization." - - create_metric(vpc_peering_active_metric["usage_name"], vpc_peering_active_metric["usage_description"]) - create_metric(vpc_peering_active_metric["limit_name"], vpc_peering_active_metric["limit_description"]) - create_metric(vpc_peering_active_metric["utilization_name"], vpc_peering_active_metric["utilization_description"]) - - create_metric(vpc_peering_metric["usage_name"], vpc_peering_metric["usage_description"]) - create_metric(vpc_peering_metric["limit_name"], vpc_peering_metric["limit_description"]) - create_metric(vpc_peering_metric["utilization_name"], vpc_peering_metric["utilization_description"]) - - return vpc_peering_active_metric, vpc_peering_metric - -# Populates data for VPC peerings per VPC and active VPC peerings per VPC -def get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric): - for project in monitored_projects_list: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_vpc_peer) - for peering in active_vpc_peerings: - write_data_to_metric(project, peering['active_peerings'], vpc_peering_active_metric["usage_name"], peering['network name']) - write_data_to_metric(project, peering['network_limit'], vpc_peering_active_metric["limit_name"], peering['network name']) - write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], vpc_peering_active_metric["utilization_name"], peering['network name']) - print("Wrote number of active VPC peerings to custom metric for project:", project) - - for peering in vpc_peerings: - write_data_to_metric(project, peering['peerings'], vpc_peering_metric["usage_name"], peering['network name']) - write_data_to_metric(project, peering['network_limit'], vpc_peering_metric["limit_name"], peering['network name']) - write_data_to_metric(project, peering['peerings'] / peering['network_limit'], vpc_peering_metric["utilization_name"], peering['network name']) - print("Wrote number of VPC peerings to custom metric for project:", project) - -# gathers number of VPC peerings, active VPC peerings and limits, returns 2 dictionaries: active_vpc_peerings and vpc_peerings (including inactive and active ones) -def gather_vpc_peerings_data(project_id, limit_list): - active_peerings_dict = [] - peerings_dict = [] - request = service.networks().list(project=project_id) - response = request.execute() - if 'items' in response: - for network in response['items']: - if 'peerings' in network: - STATE = network['peerings'][0]['state'] - if STATE == "ACTIVE": - active_peerings_count = len(network['peerings']) - else: - active_peerings_count = 0 - - peerings_count = len(network['peerings']) - else: - peerings_count = 0 - active_peerings_count = 0 - - active_d = {'project_id': project_id,'network name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)} - active_peerings_dict.append(active_d) - d = {'project_id': project_id,'network name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)} - peerings_dict.append(d) - - return active_peerings_dict, peerings_dict - -# Check if the VPC has a specific limit for a specific metric, if so, returns that limit, if not, returns the default limit -def get_limit(network_name, limit_list): - if network_name in limit_list: - return int(limit_list[limit_list.index(network_name) + 1]) - else: - if 'default_value' in limit_list: - return int(limit_list[limit_list.index('default_value') + 1]) - else: - return 0 - -# Creates the custom metrics for L4 internal forwarding Rules -def create_l4_forwarding_rules_metric(): - forwarding_rules_metric = {} - forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_usage" - forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_limit" - forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_utilization" - - forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - usage." - forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - effective limit." - forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization." - - create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) - create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) - create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) - - return forwarding_rules_metric - -def get_l4_forwarding_rules_data(forwarding_rules_metric): - - # Existing GCP Monitoring metrics for L4 Forwarding Rules - l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" - l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" - - for project in monitored_projects_list: - network_dict = get_networks(project) - - current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) - current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) - - current_quota_usage_view = customize_quota_view(current_quota_usage) - current_quota_limit_view = customize_quota_view(current_quota_limit) - - for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_l4) - write_data_to_metric(project, net['usage'], forwarding_rules_metric["usage_name"], net['network name']) - write_data_to_metric(project, net['limit'], forwarding_rules_metric["limit_name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], forwarding_rules_metric["utilization_name"], net['network name']) - - print(f"Wrote number of L4 forwarding rules to metric for projects/{project}") - -# Creates the custom metrics for L4 internal forwarding Rules per VPC Peering Group -def create_l4_forwarding_rules_ppg_metric(): - forwarding_rules_metric = {} - forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_ppg_usage" - forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_ppg_limit" - forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_ppg_utilization" - - forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - usage." - forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - effective limit." - forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - utilization." - - create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) - create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) - create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) - - return forwarding_rules_metric - -# Creates the custom metrics for L7 internal forwarding Rules per VPC Peering Group -def create_l7_forwarding_rules_ppg_metric(): - forwarding_rules_metric = {} - forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l7_ppg_usage" - forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l7_ppg_limit" - forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l7_ppg_utilization" - - forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - usage." - forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - effective limit." - forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - utilization." - - create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) - create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) - create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) - - return forwarding_rules_metric - -def create_subnet_ranges_ppg_metric(): - metric = {} - - metric["usage_name"] = "number_of_subnet_IP_ranges_usage" - metric["limit_name"] = "number_of_subnet_IP_ranges_effective_limit" - metric["utilization_name"] = "number_of_subnet_IP_ranges_utilization" - - metric["usage_description"] = "Number of Subnet Ranges per peering group - usage." - metric["limit_description"] = "Number of Subnet Ranges per peering group - effective limit." - metric["utilization_description"] = "Number of Subnet Ranges per peering group - utilization." - - create_metric(metric["usage_name"], metric["usage_description"]) - create_metric(metric["limit_name"], metric["limit_description"]) - create_metric(metric["utilization_name"], metric["utilization_description"]) - - return metric - -def create_gce_instances_ppg_metric(): - metric = {} - - metric["usage_name"] = "number_of_instances_ppg_usage" - metric["limit_name"] = "number_of_instances_ppg_limit" - metric["utilization_name"] = "number_of_instances_ppg_utilization" - - metric["usage_description"] = "Number of instances per peering group - usage." - metric["limit_description"] = "Number of instances per peering group - effective limit." - metric["utilization_description"] = "Number of instances per peering group - utilization." - - create_metric(metric["usage_name"], metric["usage_description"]) - create_metric(metric["limit_name"], metric["limit_description"]) - create_metric(metric["utilization_name"], metric["utilization_description"]) - - return metric - -# Populates data for the custom metrics for L4 internal forwarding Rules per Peering Group -def get_pgg_data(forwarding_rules_ppg_metric, usage_metric, limit_metric, limit_ppg): - - for project in monitored_projects_list: - network_dict_list = gather_peering_data(project) - # Network dict list is a list of dictionary (one for each network) - # For each network, this dictionary contains: - # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) - # peerings is a list of dictionary (one for each peered network) and contains: - # project_id, network_name, network_id - - # For each network in this GCP project - for network_dict in network_dict_list: - current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric) - current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) - - current_quota_usage_view = customize_quota_view(current_quota_usage) - current_quota_limit_view = customize_quota_view(current_quota_limit) - - usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg) - # Here we add usage and limit to the network dictionary - network_dict["usage"] = usage - network_dict["limit"] = limit - - # For every peered network, get usage and limits - for peered_network in network_dict['peerings']: - peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric)) - peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) - - usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg) - # Here we add usage and limit to the peered network dictionary - peered_network["usage"] = usage - peered_network["limit"] = limit - - count_effective_limit(project, network_dict, forwarding_rules_ppg_metric["usage_name"], forwarding_rules_ppg_metric["limit_name"], forwarding_rules_ppg_metric["utilization_name"], limit_ppg) - print(f"Wrote {forwarding_rules_ppg_metric['usage_name']} to metric for peering group {network_dict['network_name']} in {project}") - -# Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to metric -# https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit -def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg): - - if network_dict['peerings'] == []: - return - - # Get usage: Sums usage for current network + all peered networks - peering_group_usage = network_dict['usage'] - for peered_network in network_dict['peerings']: - peering_group_usage += peered_network['usage'] - - # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) - limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg)) - - # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network - limit_step2 = [] - for peered_network in network_dict['peerings']: - limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg))) - - # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 - limit_step3 = min(limit_step2) - - # Calculates effective limit: Step 4: Find maximum from step 1 and step 3 - effective_limit = max(limit_step1, limit_step3) - utilization = peering_group_usage / effective_limit - - write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name']) - write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name']) - write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) - -# Takes a project id as argument (and service for the GCP API call) and return a dictionary with the list of networks -def get_networks(project_id): - request = service.networks().list(project=project_id) - response = request.execute() - network_dict = [] - if 'items' in response: - for network in response['items']: - NETWORK = network['name'] - ID = network['id'] - d = {'project_id':project_id,'network name':NETWORK,'network id':ID} - network_dict.append(d) - return network_dict - -# gathers data for peered networks for the given project_id -def gather_peering_data(project_id): - request = service.networks().list(project=project_id) - response = request.execute() - - # list of networks in that project - network_list = [] - if 'items' in response: - for network in response['items']: - net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]} - if 'peerings' in network: - STATE = network['peerings'][0]['state'] - if STATE == "ACTIVE": - for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}" - start = peered_network['network'].find("projects/") + len('projects/') - end = peered_network['network'].find("/global") - peered_project = peered_network['network'][start:end] - peered_network_name = peered_network['network'].split("networks/")[1] - peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)} - net["peerings"].append(peered_net) - network_list.append(net) - return network_list - -def get_network_id(project_id, network_name): - request = service.networks().list(project=project_id) - response = request.execute() - - network_id = 0 - - if 'items' in response: - for network in response['items']: - if network['name'] == network_name: - network_id = network['id'] - break - - if network_id == 0: - print(f"Error: network_id not found for {network_name} in {project_id}") - - return network_id - -# retrieves quota for "type" in project_link, otherwise returns null (assume 0 for building comparison vs limits) -def get_quota_current_usage(project_link, type): - results = client.list_time_series(request={ - "name": project_link, - "filter": f'metric.type = "{type}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) - results_list = list(results) - return (results_list) - -# retrieves quota for services limits -def get_quota_current_limit(project_link, type): - results = client.list_time_series(request={ - "name": project_link, - "filter": f'metric.type = "{type}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) - results_list = list(results) - return (results_list) - -# Customize the quota output -def customize_quota_view(quota_results): - quotaViewList = [] - for result in quota_results: - quotaViewJson = {} - quotaViewJson.update(dict(result.resource.labels)) - quotaViewJson.update(dict(result.metric.labels)) - for val in result.points: - quotaViewJson.update({'value': val.value.int64_value}) - quotaViewList.append(quotaViewJson) - return (quotaViewList) - -# Takes a network dictionary and updates it with the quota usage and limits values -def set_usage_limits(network, quota_usage, quota_limit, limit_list): - if quota_usage: - for net in quota_usage: - if net['network_id'] == network['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network['usage'] = net['value'] # set network usage in dictionary - break - else: - network['usage'] = 0 # if network does not appear in GCP quotas - else: - network['usage'] = 0 # if quotas does not appear in GCP quotas - - if quota_limit: - for net in quota_limit: - if net['network_id'] == network['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network['limit'] = net['value'] # set network limit in dictionary - break - else: - if network['network name'] in limit_list: # if network limit is in the environmental variables - network['limit'] = int(limit_list[limit_list.index(network['network name']) + 1]) - else: - network['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value - else: # if quotas does not appear in GCP quotas - if network['network name'] in limit_list: - network['limit'] = int(limit_list[limit_list.index(network['network name']) + 1]) # ["default", 100, "networkname", 200] - else: - network['limit'] = int(limit_list[limit_list.index('default_value') + 1]) - -# Takes a network dictionary (with at least network_id and network_name) and returns usage and limit for that network -def get_usage_limit(network, quota_usage, quota_limit, limit_list): - usage = 0 - limit = 0 - - if quota_usage: - for net in quota_usage: - if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - usage = net['value'] # set network usage in dictionary - break - - if quota_limit: - for net in quota_limit: - if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - limit = net['value'] # set network limit in dictionary - break - else: - if network['network_name'] in limit_list: # if network limit is in the environmental variables - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) - else: - limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value - else: # if quotas does not appear in GCP quotas - if network['network_name'] in limit_list: - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200] - else: - limit = int(limit_list[limit_list.index('default_value') + 1]) - - return usage, limit - -# Writes data to Cloud Monitoring data -# Note that the monitoring_project_id here should be the monitoring project where the metrics are written -# and monitored_project_id is the monitored project, containing the network and resources -def write_data_to_metric(monitored_project_id, value, metric_name, network_name): - series = monitoring_v3.TimeSeries() - series.metric.type = f"custom.googleapis.com/{metric_name}" - series.resource.type = "global" - series.metric.labels["network_name"] = network_name - series.metric.labels["project"] = monitored_project_id - now = time.time() seconds = int(now) nanos = int((now - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}}) - point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) - series.points = [point] + interval = monitoring_v3.TimeInterval( + { + "end_time": {"seconds": seconds, "nanos": nanos}, + "start_time": {"seconds": (seconds - 86400), "nanos": nanos}, + }) + return (client, interval) + except Exception as e: + raise Exception("Error occurred creating the client: {}".format(e)) - client.create_time_series(name=monitoring_project_link, time_series=[series]) \ No newline at end of file +def create_gce_instances_metrics(): + ''' + Creates a dictionary with the name and description of 3 metrics for GCE instances per VPC network: usage, limit and utilization. + + Parameters: + None + Returns: + instance_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + instance_metric = {} + instance_metric["usage_name"] = "number_of_instances_usage" + instance_metric["limit_name"] = "number_of_instances_limit" + instance_metric["utilization_name"] = "number_of_instances_utilization" + + instance_metric["usage_description"] = "Number of instances per VPC network - usage." + instance_metric["limit_description"] = "Number of instances per VPC network - effective limit." + instance_metric["utilization_description"] = "Number of instances per VPC network - utilization." + + create_metric(instance_metric["usage_name"], instance_metric["usage_description"]) + create_metric(instance_metric["limit_name"], instance_metric["limit_description"]) + create_metric(instance_metric["utilization_name"], instance_metric["utilization_description"]) + + return instance_metric + +def create_metric(metric_name, description): + ''' + Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing + + Parameters: + metric_name (string): Name of the metric to be created + description (string): Description of the metric to be created + + Returns: + None + ''' + client = monitoring_v3.MetricServiceClient() + + metric_link = f"custom.googleapis.com/{metric_name}" + types = [] + for desc in client.list_metric_descriptors(name=monitoring_project_link): + types.append(desc.type) + + if metric_link not in types: # If the metric doesn't exist yet, then we create it + descriptor = ga_metric.MetricDescriptor() + descriptor.type = f"custom.googleapis.com/{metric_name}" + descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE + descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE + descriptor.description = description + descriptor = client.create_metric_descriptor(name=monitoring_project_link, metric_descriptor=descriptor) + print("Created {}.".format(descriptor.name)) + +def get_gce_instances_data(instance_metric): + ''' + Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. + + Parameters: + instance_metric (dictionary of string: string): metrics name and description for GCE instances per VPC Network + Returns: + None + ''' + # Existing GCP Monitoring metrics for GCE instances + metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" + metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" + + for project in monitored_projects_list: + network_dict = get_networks(project) + + current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) + + current_quota_usage_view = customize_quota_view(current_quota_usage) + current_quota_limit_view = customize_quota_view(current_quota_limit) + + for net in network_dict: + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_instances) + write_data_to_metric(project, net['usage'], instance_metric["usage_name"], net['network name']) + write_data_to_metric(project, net['limit'], instance_metric["limit_name"], net['network name']) + write_data_to_metric(project, net['usage']/ net['limit'], instance_metric["utilization_name"], net['network name']) + + print(f"Wrote number of instances to metric for projects/{project}") + + +def create_vpc_peering_metrics(): + ''' + Creates 2 dictionaries with the name and description of 3 metrics for Active VPC peering and All VPC peerings: usage, limit and utilization. + + Parameters: + None + Returns: + vpc_peering_active_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + vpc_peering_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + vpc_peering_active_metric = {} + vpc_peering_active_metric["usage_name"] = "number_of_active_vpc_peerings_usage" + vpc_peering_active_metric["limit_name"] = "number_of_active_vpc_peerings_limit" + vpc_peering_active_metric["utilization_name"] = "number_of_active_vpc_peerings_utilization" + + vpc_peering_active_metric["usage_description"] = "Number of active VPC Peerings per VPC - usage." + vpc_peering_active_metric["limit_description"] = "Number of active VPC Peerings per VPC - effective limit." + vpc_peering_active_metric["utilization_description"] = "Number of active VPC Peerings per VPC - utilization." + + vpc_peering_metric = {} + vpc_peering_metric["usage_name"] = "number_of_vpc_peerings_usage" + vpc_peering_metric["limit_name"] = "number_of_vpc_peerings_limit" + vpc_peering_metric["utilization_name"] = "number_of_vpc_peerings_utilization" + + vpc_peering_metric["usage_description"] = "Number of VPC Peerings per VPC - usage." + vpc_peering_metric["limit_description"] = "Number of VPC Peerings per VPC - effective limit." + vpc_peering_metric["utilization_description"] = "Number of VPC Peerings per VPC - utilization." + + create_metric(vpc_peering_active_metric["usage_name"], vpc_peering_active_metric["usage_description"]) + create_metric(vpc_peering_active_metric["limit_name"], vpc_peering_active_metric["limit_description"]) + create_metric(vpc_peering_active_metric["utilization_name"], vpc_peering_active_metric["utilization_description"]) + + create_metric(vpc_peering_metric["usage_name"], vpc_peering_metric["usage_description"]) + create_metric(vpc_peering_metric["limit_name"], vpc_peering_metric["limit_description"]) + create_metric(vpc_peering_metric["utilization_name"], vpc_peering_metric["utilization_description"]) + + return vpc_peering_active_metric, vpc_peering_metric + +def get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric): + ''' + Gets the data for VPC peerings (active or not) and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. + + Parameters: + vpc_peering_active_metric (dictionary of string: string): + Returns: + None + ''' + for project in monitored_projects_list: + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_vpc_peer) + for peering in active_vpc_peerings: + write_data_to_metric(project, peering['active_peerings'], vpc_peering_active_metric["usage_name"], peering['network_name']) + write_data_to_metric(project, peering['network_limit'], vpc_peering_active_metric["limit_name"], peering['network_name']) + write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], vpc_peering_active_metric["utilization_name"], peering['network_name']) + print("Wrote number of active VPC peerings to custom metric for project:", project) + + for peering in vpc_peerings: + write_data_to_metric(project, peering['peerings'], vpc_peering_metric["usage_name"], peering['network_name']) + write_data_to_metric(project, peering['network_limit'], vpc_peering_metric["limit_name"], peering['network_name']) + write_data_to_metric(project, peering['peerings'] / peering['network_limit'], vpc_peering_metric["utilization_name"], peering['network_name']) + print("Wrote number of VPC peerings to custom metric for project:", project) + +def gather_vpc_peerings_data(project_id, limit_list): + ''' + Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. + + Parameters: + project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs. + limit_list (list of string): Used to get the limit per VPC or the default limit. + Returns: + active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering. + peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering. + ''' + active_peerings_dict = [] + peerings_dict = [] + request = service.networks().list(project=project_id) + response = request.execute() + if 'items' in response: + for network in response['items']: + if 'peerings' in network: + STATE = network['peerings'][0]['state'] + if STATE == "ACTIVE": + active_peerings_count = len(network['peerings']) + else: + active_peerings_count = 0 + + peerings_count = len(network['peerings']) + else: + peerings_count = 0 + active_peerings_count = 0 + + active_d = {'project_id': project_id,'network_name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)} + active_peerings_dict.append(active_d) + d = {'project_id': project_id,'network_name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)} + peerings_dict.append(d) + + return active_peerings_dict, peerings_dict + +def get_limit(network_name, limit_list): + ''' + Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. + + Parameters: + network_name (string): Name of the VPC network. + limit_list (list of string): Used to get the limit per VPC or the default limit. + Returns: + limit (int): Limit for that VPC and that metric. + ''' + if network_name in limit_list: + return int(limit_list[limit_list.index(network_name) + 1]) + else: + if 'default_value' in limit_list: + return int(limit_list[limit_list.index('default_value') + 1]) + else: + return 0 + +def create_l4_forwarding_rules_metric(): + ''' + Creates a dictionary with the name and description of 3 metrics for L4 Internal Forwarding Rules per VPC network: usage, limit and utilization. + + Parameters: + None + Returns: + forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + forwarding_rules_metric = {} + forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_usage" + forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_limit" + forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_utilization" + + forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - usage." + forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - effective limit." + forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization." + + create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) + create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) + create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) + + return forwarding_rules_metric + +def get_l4_forwarding_rules_data(forwarding_rules_metric): + ''' + Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. + + Parameters: + forwarding_rules_metric (dictionary of string: string): metrics name and description for L4 Internal Forwarding Rules per VPC Network. + Returns: + None + ''' + # Existing GCP Monitoring metrics for L4 Forwarding Rules + l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" + l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" + + for project in monitored_projects_list: + network_dict = get_networks(project) + + current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) + + current_quota_usage_view = customize_quota_view(current_quota_usage) + current_quota_limit_view = customize_quota_view(current_quota_limit) + + for net in network_dict: + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_l4) + write_data_to_metric(project, net['usage'], forwarding_rules_metric["usage_name"], net['network name']) + write_data_to_metric(project, net['limit'], forwarding_rules_metric["limit_name"], net['network name']) + write_data_to_metric(project, net['usage']/ net['limit'], forwarding_rules_metric["utilization_name"], net['network name']) + + print(f"Wrote number of L4 forwarding rules to metric for projects/{project}") + +def create_l4_forwarding_rules_ppg_metric(): + ''' + Creates a dictionary with the name and description of 3 metrics for L4 Internal Forwarding Rules per VPC Peering Group: usage, limit and utilization. + + Parameters: + None + Returns: + forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + forwarding_rules_metric = {} + forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_ppg_usage" + forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_ppg_limit" + forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_ppg_utilization" + + forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - usage." + forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - effective limit." + forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - utilization." + + create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) + create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) + create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) + + return forwarding_rules_metric + +def create_l7_forwarding_rules_ppg_metric(): + ''' + Creates a dictionary with the name and description of 3 metrics for L7 Internal Forwarding Rules per VPC Peering Group: usage, limit and utilization. + + Parameters: + None + Returns: + forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + forwarding_rules_metric = {} + forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l7_ppg_usage" + forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l7_ppg_limit" + forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l7_ppg_utilization" + + forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - usage." + forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - effective limit." + forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - utilization." + + create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) + create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) + create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) + + return forwarding_rules_metric + +def create_subnet_ranges_ppg_metric(): + ''' + Creates a dictionary with the name and description of 3 metrics for Subnet Ranges per VPC Peering Group: usage, limit and utilization. + + Parameters: + None + Returns: + metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + metric = {} + + metric["usage_name"] = "number_of_subnet_IP_ranges_usage" + metric["limit_name"] = "number_of_subnet_IP_ranges_effective_limit" + metric["utilization_name"] = "number_of_subnet_IP_ranges_utilization" + + metric["usage_description"] = "Number of Subnet Ranges per peering group - usage." + metric["limit_description"] = "Number of Subnet Ranges per peering group - effective limit." + metric["utilization_description"] = "Number of Subnet Ranges per peering group - utilization." + + create_metric(metric["usage_name"], metric["usage_description"]) + create_metric(metric["limit_name"], metric["limit_description"]) + create_metric(metric["utilization_name"], metric["utilization_description"]) + + return metric + +def create_gce_instances_ppg_metric(): + ''' + Creates a dictionary with the name and description of 3 metrics for GCE Instances per VPC Peering Group: usage, limit and utilization. + + Parameters: + None + Returns: + metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + ''' + metric = {} + + metric["usage_name"] = "number_of_instances_ppg_usage" + metric["limit_name"] = "number_of_instances_ppg_limit" + metric["utilization_name"] = "number_of_instances_ppg_utilization" + + metric["usage_description"] = "Number of instances per peering group - usage." + metric["limit_description"] = "Number of instances per peering group - effective limit." + metric["utilization_description"] = "Number of instances per peering group - utilization." + + create_metric(metric["usage_name"], metric["usage_description"]) + create_metric(metric["limit_name"], metric["limit_description"]) + create_metric(metric["utilization_name"], metric["utilization_description"]) + + return metric + +def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): + ''' + This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored. + + Parameters: + metric_dict (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + usage_metric (string): Name of the existing GCP metric for usage per VPC network. + limit_metric (string): Name of the existing GCP metric for limit per VPC network. + limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit). + Returns: + None + ''' + for project in monitored_projects_list: + network_dict_list = gather_peering_data(project) + # Network dict list is a list of dictionary (one for each network) + # For each network, this dictionary contains: + # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) + # peerings is a list of dictionary (one for each peered network) and contains: + # project_id, network_name, network_id + + # For each network in this GCP project + for network_dict in network_dict_list: + current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric) + current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) + + current_quota_usage_view = customize_quota_view(current_quota_usage) + current_quota_limit_view = customize_quota_view(current_quota_limit) + + usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg) + # Here we add usage and limit to the network dictionary + network_dict["usage"] = usage + network_dict["limit"] = limit + + # For every peered network, get usage and limits + for peered_network in network_dict['peerings']: + peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric)) + peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) + + usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg) + # Here we add usage and limit to the peered network dictionary + peered_network["usage"] = usage + peered_network["limit"] = limit + + count_effective_limit(project, network_dict, metric_dict["usage_name"], metric_dict["limit_name"], metric_dict["utilization_name"], limit_ppg) + print(f"Wrote {metric_dict['usage_name']} to metric for peering group {network_dict['network_name']} in {project}") + +def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg): + ''' + Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. + Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit + + Parameters: + project_id (string): Project ID for the project to be analyzed. + network_dict (dictionary of string: string): Contains all required information about the network to get the usage, limit and utilization. + usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group. + limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group. + utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group. + limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit). + Returns: + None + ''' + + if network_dict['peerings'] == []: + return + + # Get usage: Sums usage for current network + all peered networks + peering_group_usage = network_dict['usage'] + for peered_network in network_dict['peerings']: + peering_group_usage += peered_network['usage'] + + # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) + limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg)) + + # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network + limit_step2 = [] + for peered_network in network_dict['peerings']: + limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg))) + + # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 + limit_step3 = min(limit_step2) + + # Calculates effective limit: Step 4: Find maximum from step 1 and step 3 + effective_limit = max(limit_step1, limit_step3) + utilization = peering_group_usage / effective_limit + + write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name']) + write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name']) + write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) + +def get_networks(project_id): + ''' + Returns a dictionary of all networks in a project. + + Parameters: + project_id (string): Project ID for the project containing the networks. + Returns: + network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) + ''' + request = service.networks().list(project=project_id) + response = request.execute() + network_dict = [] + if 'items' in response: + for network in response['items']: + NETWORK = network['name'] + ID = network['id'] + d = {'project_id':project_id,'network name':NETWORK,'network id':ID} + network_dict.append(d) + return network_dict + +def gather_peering_data(project_id): + ''' + Returns a dictionary of all peerings for all networks in a project. + + Parameters: + project_id (string): Project ID for the project containing the networks. + Returns: + network_list (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) of peered networks. + ''' + request = service.networks().list(project=project_id) + response = request.execute() + + network_list = [] + if 'items' in response: + for network in response['items']: + net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]} + if 'peerings' in network: + STATE = network['peerings'][0]['state'] + if STATE == "ACTIVE": + for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}" + start = peered_network['network'].find("projects/") + len('projects/') + end = peered_network['network'].find("/global") + peered_project = peered_network['network'][start:end] + peered_network_name = peered_network['network'].split("networks/")[1] + peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)} + net["peerings"].append(peered_net) + network_list.append(net) + return network_list + +def get_network_id(project_id, network_name): + ''' + Returns the network_id for a specific project / network name. + + Parameters: + project_id (string): Project ID for the project containing the networks. + network_name (string): Name of the network + Returns: + network_id (int): Network ID. + ''' + request = service.networks().list(project=project_id) + response = request.execute() + + network_id = 0 + + if 'items' in response: + for network in response['items']: + if network['name'] == network_name: + network_id = network['id'] + break + + if network_id == 0: + print(f"Error: network_id not found for {network_name} in {project_id}") + + return network_id + +def get_quota_current_usage(project_link, metric_name): + ''' + Retrieves quota usage for a specific metric. + + Parameters: + project_link (string): Project link. + metric_name (string): Name of the metric. + Returns: + results_list (list of string): Current usage. + ''' + results = client.list_time_series(request={ + "name": project_link, + "filter": f'metric.type = "{metric_name}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) + results_list = list(results) + return (results_list) + +def get_quota_current_limit(project_link, metric_name): + ''' + Retrieves limit for a specific metric. + + Parameters: + project_link (string): Project link. + metric_name (string): Name of the metric. + Returns: + results_list (list of string): Current limit. + ''' + results = client.list_time_series(request={ + "name": project_link, + "filter": f'metric.type = "{metric_name}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) + results_list = list(results) + return results_list + +def customize_quota_view(quota_results): + ''' + Customize the quota output for an easier parsable output. + + Parameters: + quota_results (string): Input from get_quota_current_usage or get_quota_current_limit. Contains the Current usage or limit for all networks in that project. + Returns: + quotaViewList (list of dictionaries of string: string): Current quota usage or limit. + ''' + quotaViewList = [] + for result in quota_results: + quotaViewJson = {} + quotaViewJson.update(dict(result.resource.labels)) + quotaViewJson.update(dict(result.metric.labels)) + for val in result.points: + quotaViewJson.update({'value': val.value.int64_value}) + quotaViewList.append(quotaViewJson) + return quotaViewList + +def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): + ''' + Updates the network dictionary with quota usage and limit values. + + Parameters: + network_dict (dictionary of string: string): Contains network information. + quota_usage (list of dictionaries of string: string): Current quota usage. + quota_limit (list of dictionaries of string: string): Current quota limit. + limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit). + Returns: + None + ''' + if quota_usage: + for net in quota_usage: + if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network_dict['usage'] = net['value'] # set network usage in dictionary + break + else: + network_dict['usage'] = 0 # if network does not appear in GCP quotas + else: + network_dict['usage'] = 0 # if quotas does not appear in GCP quotas + + if quota_limit: + for net in quota_limit: + if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network_dict['limit'] = net['value'] # set network limit in dictionary + break + else: + if network_dict['network name'] in limit_list: # if network limit is in the environmental variables + network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) + else: + network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value + else: # if quotas does not appear in GCP quotas + if network_dict['network name'] in limit_list: + network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) # ["default", 100, "networkname", 200] + else: + network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) + +def get_usage_limit(network, quota_usage, quota_limit, limit_list): + ''' + Returns usage and limit for a specific network and metric. + + Parameters: + network_dict (dictionary of string: string): Contains network information. + quota_usage (list of dictionaries of string: string): Current quota usage for all networks in that project. + quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project. + limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit). + Returns: + usage (int): Current usage for that network. + limit (int): Current usage for that network. + ''' + usage = 0 + limit = 0 + + if quota_usage: + for net in quota_usage: + if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + usage = net['value'] # set network usage in dictionary + break + + if quota_limit: + for net in quota_limit: + if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + limit = net['value'] # set network limit in dictionary + break + else: + if network['network_name'] in limit_list: # if network limit is in the environmental variables + limit = int(limit_list[limit_list.index(network['network_name']) + 1]) + else: + limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value + else: # if quotas does not appear in GCP quotas + if network['network_name'] in limit_list: + limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200] + else: + limit = int(limit_list[limit_list.index('default_value') + 1]) + + return usage, limit + +def write_data_to_metric(monitored_project_id, value, metric_name, network_name): + ''' + Writes data to Cloud Monitoring custom metrics. + + Parameters: + monitored_project_id: ID of the project where the resource lives (will be added as a label) + value (int): Value for the data point of the metric. + metric_name (string): Name of the metric + network_name (string): Name of the network (will be added as a label) + Returns: + usage (int): Current usage for that network. + limit (int): Current usage for that network. + ''' + series = monitoring_v3.TimeSeries() + series.metric.type = f"custom.googleapis.com/{metric_name}" + series.resource.type = "global" + series.metric.labels["network_name"] = network_name + series.metric.labels["project"] = monitored_project_id + + now = time.time() + seconds = int(now) + nanos = int((now - seconds) * 10 ** 9) + interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}}) + point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) + series.points = [point] + + client.create_time_series(name=monitoring_project_link, time_series=[series]) \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt index 5e91a8d8..037cd5b4 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt +++ b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt @@ -1,8 +1,8 @@ -regex -google-api-python-client -google-auth -google-auth-httplib2 -google-cloud-logging -google-cloud-monitoring -oauth2client -google-api-core \ No newline at end of file +regex==2022.3.2 +google-api-python-client==2.39.0 +google-auth==2.6.0 +google-auth-httplib2==0.1.0 +google-cloud-logging==3.0.0 +google-cloud-monitoring==2.9.1 +oauth2client==4.1.3 +google-api-core==2.7.0 \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 08dabef4..3c952524 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -41,7 +41,7 @@ locals { ################################################ module "project-monitoring" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" name = "monitoring" parent = "organizations/${var.organization_id}" prefix = var.prefix @@ -54,7 +54,7 @@ module "project-monitoring" { ################################################ module "service-account-function" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/iam-service-account" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/iam-service-account?ref=v14.0.0" project_id = module.project-monitoring.project_id name = "sa-dash" generate_key = false diff --git a/examples/cloud-operations/network-dashboard/tests/test.tf b/examples/cloud-operations/network-dashboard/tests/test.tf index 9b381648..ad03d9de 100644 --- a/examples/cloud-operations/network-dashboard/tests/test.tf +++ b/examples/cloud-operations/network-dashboard/tests/test.tf @@ -1,4 +1,18 @@ -# Creating test infrastructure +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ resource "google_folder" "test-net-dash" { display_name = "test-net-dash" @@ -8,7 +22,7 @@ resource "google_folder" "test-net-dash" { ##### Creating host projects, VPCs, service projects ##### module "project-hub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" name = "test-host-hub" parent = google_folder.test-net-dash.name prefix = var.prefix @@ -22,7 +36,7 @@ module "project-hub" { } module "vpc-hub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0" project_id = module.project-hub.project_id name = "vpc-hub" subnets = [ @@ -36,7 +50,7 @@ module "vpc-hub" { } module "project-svc-hub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" parent = google_folder.test-net-dash.name billing_account = var.billing_account prefix = var.prefix @@ -50,7 +64,7 @@ module "project-svc-hub" { } module "project-prod" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" name = "test-host-prod" parent = google_folder.test-net-dash.name prefix = var.prefix @@ -64,7 +78,7 @@ module "project-prod" { } module "vpc-prod" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0" project_id = module.project-prod.project_id name = "vpc-prod" subnets = [ @@ -78,7 +92,7 @@ module "vpc-prod" { } module "project-svc-prod" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" parent = google_folder.test-net-dash.name billing_account = var.billing_account prefix = var.prefix @@ -92,7 +106,7 @@ module "project-svc-prod" { } module "project-dev" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" name = "test-host-dev" parent = google_folder.test-net-dash.name prefix = var.prefix @@ -106,7 +120,7 @@ module "project-dev" { } module "vpc-dev" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0" project_id = module.project-dev.project_id name = "vpc-dev" subnets = [ @@ -120,7 +134,7 @@ module "vpc-dev" { } module "project-svc-dev" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/project" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" parent = google_folder.test-net-dash.name billing_account = var.billing_account prefix = var.prefix @@ -136,26 +150,26 @@ module "project-svc-dev" { ##### Creating VPC peerings ##### module "hub-to-prod-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" local_network = module.vpc-hub.self_link peer_network = module.vpc-prod.self_link } module "prod-to-hub-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" local_network = module.vpc-prod.self_link peer_network = module.vpc-hub.self_link depends_on = [module.hub-to-prod-peering] } module "hub-to-dev-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" local_network = module.vpc-hub.self_link peer_network = module.vpc-dev.self_link } module "dev-to-hub-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric///modules/net-vpc-peering" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" local_network = module.vpc-dev.self_link peer_network = module.vpc-hub.self_link depends_on = [module.hub-to-dev-peering] From 9a076553a5f014c9818dd83e885c8077c8bbf4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 16:36:36 +0100 Subject: [PATCH 03/23] Refactoring Cloud Function code: added metrics.yaml file to create all metrics. --- .../network-dashboard/cloud-function/main.py | 265 +++--------------- .../cloud-function/metrics.yaml | 93 ++++++ .../cloud-function/requirements.txt | 3 +- 3 files changed, 140 insertions(+), 221 deletions(-) create mode 100644 examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 407e99a1..4889980c 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -22,8 +22,8 @@ import os import google.api_core import re import random +import yaml -# monitored_projects_list = os.environ.get("monitored_projects_list").split(",") # list of projects from which function will get quotas information monitoring_project_id = os.environ.get("monitoring_project_id") # project where the metrics and dahsboards will be created monitoring_project_link = f"projects/{monitoring_project_id}" @@ -39,54 +39,49 @@ limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",") limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",") limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",") -def quotas(request): +def main(event, context): ''' Cloud Function Entry point, called by the scheduler. Parameters: - request: for now, the Cloud Function is triggered by an HTTP trigger and this request correspond to the HTTP triggering request. + event: Not used for now (Pubsub trigger) + context: Not used for now (Pubsub trigger) Returns: 'Function executed successfully' ''' global client, interval client, interval = create_client() - instance_metric = create_gce_instances_metrics() - get_gce_instances_data(instance_metric) + metrics_dict = create_metrics() - vpc_peering_active_metric, vpc_peering_metric = create_vpc_peering_metrics() - get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric) - - forwarding_rules_metric = create_l4_forwarding_rules_metric() - get_l4_forwarding_rules_data(forwarding_rules_metric) + # Per Network metrics + get_gce_instances_data(metrics_dict) + get_vpc_peering_data(metrics_dict) + get_l4_forwarding_rules_data(metrics_dict) # Existing GCP Monitoring metrics for L4 Forwarding Rules per Network l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" - l4_forwarding_rules_ppg_metric = create_l4_forwarding_rules_ppg_metric() - get_pgg_data(l4_forwarding_rules_ppg_metric, l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg) # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" - l7_forwarding_rules_ppg_metric = create_l7_forwarding_rules_ppg_metric() - get_pgg_data(l7_forwarding_rules_ppg_metric, l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg) # Existing GCP Monitoring metrics for Subnet Ranges per Network subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" - subnet_ranges_ppg_metric = create_subnet_ranges_ppg_metric() - get_pgg_data(subnet_ranges_ppg_metric, subnet_ranges_usage, subnet_ranges_limit, limit_subnets) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, limit_subnets) # Existing GCP Monitoring metrics for GCE per Network gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - gce_instances_metric = create_gce_instances_ppg_metric() - get_pgg_data(gce_instances_metric, gce_instances_usage, gce_instances_limit, limit_instances_ppg) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, limit_instances_ppg) return 'Function executed successfully' @@ -114,29 +109,19 @@ def create_client(): except Exception as e: raise Exception("Error occurred creating the client: {}".format(e)) -def create_gce_instances_metrics(): - ''' - Creates a dictionary with the name and description of 3 metrics for GCE instances per VPC network: usage, limit and utilization. +def create_metrics(): + with open("metrics.yaml", 'r') as stream: + try: + metrics_dict = yaml.safe_load(stream) - Parameters: - None - Returns: - instance_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - instance_metric = {} - instance_metric["usage_name"] = "number_of_instances_usage" - instance_metric["limit_name"] = "number_of_instances_limit" - instance_metric["utilization_name"] = "number_of_instances_utilization" + for metric_list in metrics_dict.values(): + for metric in metric_list.values(): + for sub_metric in metric.values(): + create_metric(sub_metric["name"], sub_metric["description"]) - instance_metric["usage_description"] = "Number of instances per VPC network - usage." - instance_metric["limit_description"] = "Number of instances per VPC network - effective limit." - instance_metric["utilization_description"] = "Number of instances per VPC network - utilization." - - create_metric(instance_metric["usage_name"], instance_metric["usage_description"]) - create_metric(instance_metric["limit_name"], instance_metric["limit_description"]) - create_metric(instance_metric["utilization_name"], instance_metric["utilization_description"]) - - return instance_metric + return metrics_dict + except yaml.YAMLError as exc: + print(exc) def create_metric(metric_name, description): ''' @@ -165,12 +150,12 @@ def create_metric(metric_name, description): descriptor = client.create_metric_descriptor(name=monitoring_project_link, metric_descriptor=descriptor) print("Created {}.".format(descriptor.name)) -def get_gce_instances_data(instance_metric): +def get_gce_instances_data(metrics_dict): ''' Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. Parameters: - instance_metric (dictionary of string: string): metrics name and description for GCE instances per VPC Network + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions Returns: None ''' @@ -189,72 +174,34 @@ def get_gce_instances_data(instance_metric): for net in network_dict: set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_instances) - write_data_to_metric(project, net['usage'], instance_metric["usage_name"], net['network name']) - write_data_to_metric(project, net['limit'], instance_metric["limit_name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], instance_metric["utilization_name"], net['network name']) + write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["instance_per_network"]["usage"]["name"], net['network name']) + write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["limit"]["name"], net['network name']) + write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["utilization"]["name"], net['network name']) print(f"Wrote number of instances to metric for projects/{project}") -def create_vpc_peering_metrics(): +def get_vpc_peering_data(metrics_dict): ''' - Creates 2 dictionaries with the name and description of 3 metrics for Active VPC peering and All VPC peerings: usage, limit and utilization. + Gets the data for VPC peerings (active or not) and writes it to the metric defined (vpc_peering_active_metric and vpc_peering_metric). Parameters: - None - Returns: - vpc_peering_active_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - vpc_peering_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - vpc_peering_active_metric = {} - vpc_peering_active_metric["usage_name"] = "number_of_active_vpc_peerings_usage" - vpc_peering_active_metric["limit_name"] = "number_of_active_vpc_peerings_limit" - vpc_peering_active_metric["utilization_name"] = "number_of_active_vpc_peerings_utilization" - - vpc_peering_active_metric["usage_description"] = "Number of active VPC Peerings per VPC - usage." - vpc_peering_active_metric["limit_description"] = "Number of active VPC Peerings per VPC - effective limit." - vpc_peering_active_metric["utilization_description"] = "Number of active VPC Peerings per VPC - utilization." - - vpc_peering_metric = {} - vpc_peering_metric["usage_name"] = "number_of_vpc_peerings_usage" - vpc_peering_metric["limit_name"] = "number_of_vpc_peerings_limit" - vpc_peering_metric["utilization_name"] = "number_of_vpc_peerings_utilization" - - vpc_peering_metric["usage_description"] = "Number of VPC Peerings per VPC - usage." - vpc_peering_metric["limit_description"] = "Number of VPC Peerings per VPC - effective limit." - vpc_peering_metric["utilization_description"] = "Number of VPC Peerings per VPC - utilization." - - create_metric(vpc_peering_active_metric["usage_name"], vpc_peering_active_metric["usage_description"]) - create_metric(vpc_peering_active_metric["limit_name"], vpc_peering_active_metric["limit_description"]) - create_metric(vpc_peering_active_metric["utilization_name"], vpc_peering_active_metric["utilization_description"]) - - create_metric(vpc_peering_metric["usage_name"], vpc_peering_metric["usage_description"]) - create_metric(vpc_peering_metric["limit_name"], vpc_peering_metric["limit_description"]) - create_metric(vpc_peering_metric["utilization_name"], vpc_peering_metric["utilization_description"]) - - return vpc_peering_active_metric, vpc_peering_metric - -def get_vpc_peering_data(vpc_peering_active_metric, vpc_peering_metric): - ''' - Gets the data for VPC peerings (active or not) and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. - - Parameters: - vpc_peering_active_metric (dictionary of string: string): + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions Returns: None ''' for project in monitored_projects_list: active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_vpc_peer) for peering in active_vpc_peerings: - write_data_to_metric(project, peering['active_peerings'], vpc_peering_active_metric["usage_name"], peering['network_name']) - write_data_to_metric(project, peering['network_limit'], vpc_peering_active_metric["limit_name"], peering['network_name']) - write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], vpc_peering_active_metric["utilization_name"], peering['network_name']) + write_data_to_metric(project, peering['active_peerings'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["usage"]["name"], peering['network_name']) + write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["limit"]["name"], peering['network_name']) + write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["utilization"]["name"], peering['network_name']) print("Wrote number of active VPC peerings to custom metric for project:", project) for peering in vpc_peerings: - write_data_to_metric(project, peering['peerings'], vpc_peering_metric["usage_name"], peering['network_name']) - write_data_to_metric(project, peering['network_limit'], vpc_peering_metric["limit_name"], peering['network_name']) - write_data_to_metric(project, peering['peerings'] / peering['network_limit'], vpc_peering_metric["utilization_name"], peering['network_name']) + write_data_to_metric(project, peering['peerings'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["usage"]["name"], peering['network_name']) + write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["limit"]["name"], peering['network_name']) + write_data_to_metric(project, peering['peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["utilization"]["name"], peering['network_name']) print("Wrote number of VPC peerings to custom metric for project:", project) def gather_vpc_peerings_data(project_id, limit_list): @@ -311,36 +258,12 @@ def get_limit(network_name, limit_list): else: return 0 -def create_l4_forwarding_rules_metric(): - ''' - Creates a dictionary with the name and description of 3 metrics for L4 Internal Forwarding Rules per VPC network: usage, limit and utilization. - - Parameters: - None - Returns: - forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - forwarding_rules_metric = {} - forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_usage" - forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_limit" - forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_utilization" - - forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - usage." - forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - effective limit." - forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization." - - create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) - create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) - create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) - - return forwarding_rules_metric - -def get_l4_forwarding_rules_data(forwarding_rules_metric): +def get_l4_forwarding_rules_data(metrics_dict): ''' Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. Parameters: - forwarding_rules_metric (dictionary of string: string): metrics name and description for L4 Internal Forwarding Rules per VPC Network. + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions Returns: None ''' @@ -359,110 +282,12 @@ def get_l4_forwarding_rules_data(forwarding_rules_metric): for net in network_dict: set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_l4) - write_data_to_metric(project, net['usage'], forwarding_rules_metric["usage_name"], net['network name']) - write_data_to_metric(project, net['limit'], forwarding_rules_metric["limit_name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], forwarding_rules_metric["utilization_name"], net['network name']) + write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name']) + write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name']) + write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["utilization"]["name"], net['network name']) print(f"Wrote number of L4 forwarding rules to metric for projects/{project}") -def create_l4_forwarding_rules_ppg_metric(): - ''' - Creates a dictionary with the name and description of 3 metrics for L4 Internal Forwarding Rules per VPC Peering Group: usage, limit and utilization. - - Parameters: - None - Returns: - forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - forwarding_rules_metric = {} - forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l4_ppg_usage" - forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l4_ppg_limit" - forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l4_ppg_utilization" - - forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - usage." - forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - effective limit." - forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l4 Load Balancers per peering group - utilization." - - create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) - create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) - create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) - - return forwarding_rules_metric - -def create_l7_forwarding_rules_ppg_metric(): - ''' - Creates a dictionary with the name and description of 3 metrics for L7 Internal Forwarding Rules per VPC Peering Group: usage, limit and utilization. - - Parameters: - None - Returns: - forwarding_rules_metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - forwarding_rules_metric = {} - forwarding_rules_metric["usage_name"] = "internal_forwarding_rules_l7_ppg_usage" - forwarding_rules_metric["limit_name"] = "internal_forwarding_rules_l7_ppg_limit" - forwarding_rules_metric["utilization_name"] = "internal_forwarding_rules_l7_ppg_utilization" - - forwarding_rules_metric["usage_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - usage." - forwarding_rules_metric["limit_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - effective limit." - forwarding_rules_metric["utilization_description"] = "Number of Internal Forwarding Rules for Internal l7 Load Balancers per peering group - utilization." - - create_metric(forwarding_rules_metric["usage_name"], forwarding_rules_metric["usage_description"]) - create_metric(forwarding_rules_metric["limit_name"], forwarding_rules_metric["limit_description"]) - create_metric(forwarding_rules_metric["utilization_name"], forwarding_rules_metric["utilization_description"]) - - return forwarding_rules_metric - -def create_subnet_ranges_ppg_metric(): - ''' - Creates a dictionary with the name and description of 3 metrics for Subnet Ranges per VPC Peering Group: usage, limit and utilization. - - Parameters: - None - Returns: - metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - metric = {} - - metric["usage_name"] = "number_of_subnet_IP_ranges_usage" - metric["limit_name"] = "number_of_subnet_IP_ranges_effective_limit" - metric["utilization_name"] = "number_of_subnet_IP_ranges_utilization" - - metric["usage_description"] = "Number of Subnet Ranges per peering group - usage." - metric["limit_description"] = "Number of Subnet Ranges per peering group - effective limit." - metric["utilization_description"] = "Number of Subnet Ranges per peering group - utilization." - - create_metric(metric["usage_name"], metric["usage_description"]) - create_metric(metric["limit_name"], metric["limit_description"]) - create_metric(metric["utilization_name"], metric["utilization_description"]) - - return metric - -def create_gce_instances_ppg_metric(): - ''' - Creates a dictionary with the name and description of 3 metrics for GCE Instances per VPC Peering Group: usage, limit and utilization. - - Parameters: - None - Returns: - metric (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) - ''' - metric = {} - - metric["usage_name"] = "number_of_instances_ppg_usage" - metric["limit_name"] = "number_of_instances_ppg_limit" - metric["utilization_name"] = "number_of_instances_ppg_utilization" - - metric["usage_description"] = "Number of instances per peering group - usage." - metric["limit_description"] = "Number of instances per peering group - effective limit." - metric["utilization_description"] = "Number of instances per peering group - utilization." - - create_metric(metric["usage_name"], metric["usage_description"]) - create_metric(metric["limit_name"], metric["limit_description"]) - create_metric(metric["utilization_name"], metric["utilization_description"]) - - return metric - def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): ''' This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored. @@ -506,8 +331,8 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): peered_network["usage"] = usage peered_network["limit"] = limit - count_effective_limit(project, network_dict, metric_dict["usage_name"], metric_dict["limit_name"], metric_dict["utilization_name"], limit_ppg) - print(f"Wrote {metric_dict['usage_name']} to metric for peering group {network_dict['network_name']} in {project}") + count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_ppg) + print(f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}") def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg): ''' diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml new file mode 100644 index 00000000..d576034f --- /dev/null +++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -0,0 +1,93 @@ +--- +metrics_per_network: + instance_per_network: + usage: + name: number_of_instances_usage + description: Number of instances per VPC network - usage. + limit: + name: number_of_instances_limit + description: Number of instances per VPC network - limit. + utilization: + name: number_of_instances_utilization + description: Number of instances per VPC network - utilization. + vpc_peering_active_per_network: + usage: + name: number_of_active_vpc_peerings_usage + description: Number of active VPC Peerings per VPC - usage. + limit: + name: number_of_active_vpc_peerings_limit + description: Number of active VPC Peerings per VPC - limit. + utilization: + name: number_of_active_vpc_peerings_utilization + description: Number of active VPC Peerings per VPC - utilization. + vpc_peering_per_network: + usage: + name: number_of_vpc_peerings_usage + description: Number of VPC Peerings per VPC - usage. + limit: + name: number_of_vpc_peerings_limit + description: Number of VPC Peerings per VPC - limit. + utilization: + name: number_of_vpc_peerings_utilization + description: Number of VPC Peerings per VPC - utilization. + l4_forwarding_rules_per_network: + usage: + name: internal_forwarding_rules_l4_usage + description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - usage. + limit: + name: internal_forwarding_rules_l4_limit + description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - limit. + utilization: + name: internal_forwarding_rules_l4_utilization + description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization. + l7_forwarding_rules_per_network: + usage: + name: internal_forwarding_rules_l7_usage + description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per network - usage. + limit: + name: internal_forwarding_rules_l7_limit + description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per network - effective limit. + utilization: + name: internal_forwarding_rules_l7_utilization + description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per Vnetwork - utilization. +metrics_per_peering_group: + l4_forwarding_rules_per_peering_group: + usage: + name: internal_forwarding_rules_l4_ppg_usage + description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - usage. + limit: + name: internal_forwarding_rules_l4_ppg_limit + description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - effective limit. + utilization: + name: internal_forwarding_rules_l4_ppg_utilization + description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - utilization. + l7_forwarding_rules_per_peering_group: + usage: + name: internal_forwarding_rules_l7_ppg_usage + description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - usage. + limit: + name: internal_forwarding_rules_l7_ppg_limit + description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - effective limit. + utilization: + name: internal_forwarding_rules_l7_ppg_utilization + description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - utilization. + subnet_ranges_per_peering_group: + usage: + name: number_of_subnet_IP_ranges_usage + description: Number of Subnet Ranges per peering group - usage. + limit: + name: number_of_subnet_IP_ranges_limit + description: Number of Subnet Ranges per peering group - effective limit. + utilization: + name: number_of_subnet_IP_ranges_utilization + description: Number of Subnet Ranges per peering group - utilization. + instance_per_peering_group: + usage: + name: number_of_instances_ppg_usage + description: Number of instances per peering group - usage. + limit: + name: number_of_instances_ppg_limit + description: Number of instances per peering group - limit. + utilization: + name: number_of_instances_ppg_utilization + description: Number of instances per peering group - utilization. \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt index 037cd5b4..0888969c 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt +++ b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt @@ -5,4 +5,5 @@ google-auth-httplib2==0.1.0 google-cloud-logging==3.0.0 google-cloud-monitoring==2.9.1 oauth2client==4.1.3 -google-api-core==2.7.0 \ No newline at end of file +google-api-core==2.7.0 +PyYAML==6.0 \ No newline at end of file From 5a085f41478142cbf39cd2f9b9d11652d7574060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 16:36:56 +0100 Subject: [PATCH 04/23] Using Cloud Function Fabric Terraform module --- .../network-dashboard/main.tf | 106 +++++++++--------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 3c952524..37ac388f 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -80,57 +80,62 @@ module "service-account-function" { # Cloud Function configuration (& Scheduler) # ################################################ -# Create an app engine application (required for Cloud Scheduler) -resource "google_app_engine_application" "scheduler_app" { - project = module.project-monitoring.project_id - # "europe-west1" is called "europe-west" and "us-central1" is "us-central" for App Engine, see https://cloud.google.com/appengine/docs/locations - location_id = var.region == "europe-west1" || var.region == "us-central1" ? substr(var.region, 0, length(var.region) - 1) : var.region +module "pubsub" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/pubsub?ref=v14.0.0" + project_id = module.project-monitoring.project_id + name = "network-dashboard-pubsub" + subscriptions = { + "network-dashboard-pubsub-default" = null + } + # the Cloud Scheduler robot service account already has pubsub.topics.publish + # at the project level via roles/cloudscheduler.serviceAgent } -# Create a storage bucket for the Cloud Function's code -resource "google_storage_bucket" "bucket" { - name = "net-quotas-bucket" - location = "EU" - project = module.project-monitoring.project_id +resource "google_cloud_scheduler_job" "job" { + project = module.project-monitoring.project_id + region = var.region + name = "network-dashboard-scheduler" + schedule = var.schedule_cron + time_zone = "UTC" + pubsub_target { + topic_name = module.pubsub.topic.id + data = base64encode("test") + } } -data "archive_file" "file" { - type = "zip" - source_dir = "cloud-function" - output_path = "cloud-function.zip" - depends_on = [google_storage_bucket.bucket] +# Random ID to re-deploy the Cloud Function with every Terraform run +resource "random_pet" "random" { + length = 1 } -resource "google_storage_bucket_object" "archive" { - # md5 hash in the bucket object name to redeploy the Cloud Function when the code is modified - name = format("cloud-function#%s", data.archive_file.file.output_md5) - bucket = google_storage_bucket.bucket.name - source = "cloud-function.zip" - depends_on = [data.archive_file.file] -} +module "cloud-function" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/cloud-function?ref=v14.0.0" + project_id = module.project-monitoring.project_id + name = "network-dashboard-cloud-function" + bucket_name = "network-dashboard-bucket-${random_pet.random.id}" + bucket_config = { + location = var.region + lifecycle_delete_age = null + } -resource "google_cloudfunctions_function" "function_quotas" { - name = "function-quotas" - project = module.project-monitoring.project_id - region = var.region - description = "Function which creates metric to show number, limit and utlizitation." - runtime = "python39" - - available_memory_mb = 512 - source_archive_bucket = google_storage_bucket.bucket.name - source_archive_object = google_storage_bucket_object.archive.name - service_account_email = module.service-account-function.email - - timeout = 180 - entry_point = "quotas" - trigger_http = true + bundle_config = { + source_dir = "cloud-function" + output_path = "cloud-function.zip" + excludes = null + } + function_config = { + timeout = 180 + entry_point = "main" + runtime = "python39" + instances = 1 + memory = 256 + } environment_variables = { monitored_projects_list = local.projects monitoring_project_id = module.project-monitoring.project_id - LIMIT_SUBNETS = local.limit_subnets LIMIT_INSTANCES = local.limit_instances LIMIT_INSTANCES_PPG = local.limit_instances_ppg @@ -140,24 +145,13 @@ resource "google_cloudfunctions_function" "function_quotas" { LIMIT_L4_PPG = local.limit_l4_ppg LIMIT_L7_PPG = local.limit_l7_ppg } -} -resource "google_cloud_scheduler_job" "job" { - name = "scheduler-net-dash" - project = module.project-monitoring.project_id - region = var.region - description = "Cloud Scheduler job to trigger the Networking Dashboard Cloud Function" - schedule = var.schedule_cron + service_account = module.service-account-function.email - retry_config { - retry_count = 1 - } - - http_target { - http_method = "POST" - uri = google_cloudfunctions_function.function_quotas.https_trigger_url - # We could pass useful data in the body later - body = base64encode("{\"foo\":\"bar\"}") + trigger_config = { + event = "google.pubsub.topic.publish" + resource = module.pubsub.topic.id + retry = null } } @@ -167,7 +161,7 @@ resource "google_cloud_scheduler_job" "job" { resource "google_cloudfunctions_function_iam_member" "invoker" { project = module.project-monitoring.project_id region = var.region - cloud_function = google_cloudfunctions_function.function_quotas.name + cloud_function = module.cloud-function.function_name role = "roles/cloudfunctions.invoker" member = "allUsers" @@ -180,4 +174,4 @@ resource "google_cloudfunctions_function_iam_member" "invoker" { resource "google_monitoring_dashboard" "dashboard" { dashboard_json = file("${path.module}/dashboards/quotas-utilization.json") project = module.project-monitoring.project_id -} +} \ No newline at end of file From a1fdb73a96a5b8c15843f8b58bb3b49253702b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 16:38:24 +0100 Subject: [PATCH 05/23] formatting --- .../network-dashboard/main.tf | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 37ac388f..5ba3a4f7 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -100,7 +100,7 @@ resource "google_cloud_scheduler_job" "job" { pubsub_target { topic_name = module.pubsub.topic.id - data = base64encode("test") + data = base64encode("test") } } @@ -110,7 +110,7 @@ resource "random_pet" "random" { } module "cloud-function" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/cloud-function?ref=v14.0.0" + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/cloud-function?ref=v14.0.0" project_id = module.project-monitoring.project_id name = "network-dashboard-cloud-function" bucket_name = "network-dashboard-bucket-${random_pet.random.id}" @@ -126,24 +126,24 @@ module "cloud-function" { } function_config = { - timeout = 180 + timeout = 180 entry_point = "main" - runtime = "python39" - instances = 1 - memory = 256 + runtime = "python39" + instances = 1 + memory = 256 } environment_variables = { monitored_projects_list = local.projects monitoring_project_id = module.project-monitoring.project_id - LIMIT_SUBNETS = local.limit_subnets - LIMIT_INSTANCES = local.limit_instances - LIMIT_INSTANCES_PPG = local.limit_instances_ppg - LIMIT_VPC_PEER = local.limit_vpc_peer - LIMIT_L4 = local.limit_l4 - LIMIT_L7 = local.limit_l7 - LIMIT_L4_PPG = local.limit_l4_ppg - LIMIT_L7_PPG = local.limit_l7_ppg + LIMIT_SUBNETS = local.limit_subnets + LIMIT_INSTANCES = local.limit_instances + LIMIT_INSTANCES_PPG = local.limit_instances_ppg + LIMIT_VPC_PEER = local.limit_vpc_peer + LIMIT_L4 = local.limit_l4 + LIMIT_L7 = local.limit_l7 + LIMIT_L4_PPG = local.limit_l4_ppg + LIMIT_L7_PPG = local.limit_l7_ppg } service_account = module.service-account-function.email From 13244d8904eb9833265bf29d706cf7e862d74660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 17:04:31 +0100 Subject: [PATCH 06/23] Removing global variables. --- .../network-dashboard/cloud-function/main.py | 8 ++++++-- .../network-dashboard/cloud-function/metrics.yaml | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 4889980c..f8dc2675 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -49,8 +49,6 @@ def main(event, context): Returns: 'Function executed successfully' ''' - global client, interval - client, interval = create_client() metrics_dict = create_metrics() @@ -462,6 +460,8 @@ def get_quota_current_usage(project_link, metric_name): Returns: results_list (list of string): Current usage. ''' + client, interval = create_client() + results = client.list_time_series(request={ "name": project_link, "filter": f'metric.type = "{metric_name}"', @@ -481,6 +481,8 @@ def get_quota_current_limit(project_link, metric_name): Returns: results_list (list of string): Current limit. ''' + client, interval = create_client() + results = client.list_time_series(request={ "name": project_link, "filter": f'metric.type = "{metric_name}"', @@ -600,6 +602,8 @@ def write_data_to_metric(monitored_project_id, value, metric_name, network_name) usage (int): Current usage for that network. limit (int): Current usage for that network. ''' + client = monitoring_v3.MetricServiceClient() + series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml index d576034f..233dc9be 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -1,3 +1,18 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --- metrics_per_network: instance_per_network: From 6f6b0796da61ac478514fefa715ec2d5886249ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 17:28:17 +0100 Subject: [PATCH 07/23] Variables to uppercase. --- .../network-dashboard/cloud-function/main.py | 54 ++++++++++--------- .../network-dashboard/main.tf | 6 +-- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index f8dc2675..4680e04f 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -24,20 +24,22 @@ import re import random import yaml -monitored_projects_list = os.environ.get("monitored_projects_list").split(",") # list of projects from which function will get quotas information -monitoring_project_id = os.environ.get("monitoring_project_id") # project where the metrics and dahsboards will be created -monitoring_project_link = f"projects/{monitoring_project_id}" +# list of projects from which function will get quotas information +MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",") +# project where the metrics and dahsboards will be created +MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID") +MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}" service = discovery.build('compute', 'v1') # DEFAULT LIMITS: -limit_vpc_peer = os.environ.get("LIMIT_VPC_PEER").split(",") -limit_l4 = os.environ.get("LIMIT_L4").split(",") -limit_l7 = os.environ.get("LIMIT_L7").split(",") -limit_instances = os.environ.get("LIMIT_INSTANCES").split(",") -limit_instances_ppg = os.environ.get("LIMIT_INSTANCES_PPG").split(",") -limit_subnets = os.environ.get("LIMIT_SUBNETS").split(",") -limit_l4_ppg = os.environ.get("LIMIT_L4_PPG").split(",") -limit_l7_ppg = os.environ.get("LIMIT_L7_PPG").split(",") +LIMIT_VPC_PEER = os.environ.get("LIMIT_VPC_PEER").split(",") +LIMIT_L4 = os.environ.get("LIMIT_L4").split(",") +LIMIT_L7 = os.environ.get("LIMIT_L7").split(",") +LIMIT_INSTANCES = os.environ.get("LIMIT_INSTANCES").split(",") +LIMIT_INSTANCES_PPG = os.environ.get("LIMIT_INSTANCES_PPG").split(",") +LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",") +LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",") +LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",") def main(event, context): ''' @@ -61,25 +63,25 @@ def main(event, context): l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" - get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, limit_l4_ppg) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, LIMIT_L4_PPG) # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" - get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, limit_l7_ppg) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, LIMIT_L7_PPG) # Existing GCP Monitoring metrics for Subnet Ranges per Network subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" - get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, limit_subnets) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, LIMIT_SUBNETS) # Existing GCP Monitoring metrics for GCE per Network gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, limit_instances_ppg) + get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, LIMIT_INSTANCES_PPG) return 'Function executed successfully' @@ -136,7 +138,7 @@ def create_metric(metric_name, description): metric_link = f"custom.googleapis.com/{metric_name}" types = [] - for desc in client.list_metric_descriptors(name=monitoring_project_link): + for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): types.append(desc.type) if metric_link not in types: # If the metric doesn't exist yet, then we create it @@ -145,7 +147,7 @@ def create_metric(metric_name, description): descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE descriptor.description = description - descriptor = client.create_metric_descriptor(name=monitoring_project_link, metric_descriptor=descriptor) + descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, metric_descriptor=descriptor) print("Created {}.".format(descriptor.name)) def get_gce_instances_data(metrics_dict): @@ -161,7 +163,7 @@ def get_gce_instances_data(metrics_dict): metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - for project in monitored_projects_list: + for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) @@ -171,7 +173,7 @@ def get_gce_instances_data(metrics_dict): current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_instances) + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_INSTANCES) write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["instance_per_network"]["usage"]["name"], net['network name']) write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["limit"]["name"], net['network name']) write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["utilization"]["name"], net['network name']) @@ -188,8 +190,8 @@ def get_vpc_peering_data(metrics_dict): Returns: None ''' - for project in monitored_projects_list: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_vpc_peer) + for project in MONITORED_PROJECTS_LIST: + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, LIMIT_VPC_PEER) for peering in active_vpc_peerings: write_data_to_metric(project, peering['active_peerings'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["usage"]["name"], peering['network_name']) write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["limit"]["name"], peering['network_name']) @@ -269,7 +271,7 @@ def get_l4_forwarding_rules_data(metrics_dict): l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" - for project in monitored_projects_list: + for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) @@ -279,7 +281,7 @@ def get_l4_forwarding_rules_data(metrics_dict): current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, limit_l4) + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_L4) write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name']) write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name']) write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["utilization"]["name"], net['network name']) @@ -291,14 +293,14 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored. Parameters: - metric_dict (dictionary of string: string): A dictionary with the metric names and description, that will be used later on to create the metrics in create_metric(metric_name, description) + metric_dict (dictionary of string: string): A dictionary with the metric names and description, that will be used to populate the metrics usage_metric (string): Name of the existing GCP metric for usage per VPC network. limit_metric (string): Name of the existing GCP metric for limit per VPC network. limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit). Returns: None ''' - for project in monitored_projects_list: + for project in MONITORED_PROJECTS_LIST: network_dict_list = gather_peering_data(project) # Network dict list is a list of dictionary (one for each network) # For each network, this dictionary contains: @@ -617,4 +619,4 @@ def write_data_to_metric(monitored_project_id, value, metric_name, network_name) point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) series.points = [point] - client.create_time_series(name=monitoring_project_link, time_series=[series]) \ No newline at end of file + client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 5ba3a4f7..4a797900 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -134,8 +134,8 @@ module "cloud-function" { } environment_variables = { - monitored_projects_list = local.projects - monitoring_project_id = module.project-monitoring.project_id + MONITORED_PROJECTS_LIST = local.projects + MONITORING_PROJECT_ID = module.project-monitoring.project_id LIMIT_SUBNETS = local.limit_subnets LIMIT_INSTANCES = local.limit_instances LIMIT_INSTANCES_PPG = local.limit_instances_ppg @@ -174,4 +174,4 @@ resource "google_cloudfunctions_function_iam_member" "invoker" { resource "google_monitoring_dashboard" "dashboard" { dashboard_json = file("${path.module}/dashboards/quotas-utilization.json") project = module.project-monitoring.project_id -} \ No newline at end of file +} From 65172031f0f525adfe6d56aa050aab606d6ce8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 20:08:58 +0100 Subject: [PATCH 08/23] refactoring main.py --- .../network-dashboard/cloud-function/main.py | 398 ++++++++++++------ .../network-dashboard/main.tf | 12 - 2 files changed, 270 insertions(+), 140 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 4680e04f..ccbc05e7 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -14,20 +14,17 @@ # limitations under the License. # +import time +import os +import yaml from google.cloud import monitoring_v3 from googleapiclient import discovery from google.api import metric_pb2 as ga_metric -import time -import os -import google.api_core -import re -import random -import yaml # list of projects from which function will get quotas information MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",") # project where the metrics and dahsboards will be created -MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID") +MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID") MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}" service = discovery.build('compute', 'v1') @@ -41,6 +38,17 @@ LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",") LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",") LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",") +# Existing GCP metrics per network +L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" +L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" +L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" +L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" +SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" +SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" +GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage" +GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit" + + def main(event, context): ''' Cloud Function Entry point, called by the scheduler. @@ -59,32 +67,31 @@ def main(event, context): get_vpc_peering_data(metrics_dict) get_l4_forwarding_rules_data(metrics_dict) - # Existing GCP Monitoring metrics for L4 Forwarding Rules per Network - l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" - l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" + get_pgg_data( + metrics_dict["metrics_per_peering_group"] + ["l4_forwarding_rules_per_peering_group"], + L4_FORWARDING_RULES_USAGE_METRIC, L4_FORWARDING_RULES_LIMIT_METRIC, + LIMIT_L4_PPG) - get_pgg_data(metrics_dict["metrics_per_peering_group"]["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_usage, l4_forwarding_rules_limit, LIMIT_L4_PPG) + get_pgg_data( + metrics_dict["metrics_per_peering_group"] + ["l7_forwarding_rules_per_peering_group"], + L7_FORWARDING_RULES_USAGE_METRIC, L7_FORWARDING_RULES_LIMIT_METRIC, + LIMIT_L7_PPG) - # Existing GCP Monitoring metrics for L7 Forwarding Rules per Network - l7_forwarding_rules_usage = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" - l7_forwarding_rules_limit = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" + get_pgg_data( + metrics_dict["metrics_per_peering_group"] + ["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC, + SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS) - get_pgg_data(metrics_dict["metrics_per_peering_group"]["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_usage, l7_forwarding_rules_limit, LIMIT_L7_PPG) - - # Existing GCP Monitoring metrics for Subnet Ranges per Network - subnet_ranges_usage = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" - subnet_ranges_limit = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" - - get_pgg_data(metrics_dict["metrics_per_peering_group"]["subnet_ranges_per_peering_group"], subnet_ranges_usage, subnet_ranges_limit, LIMIT_SUBNETS) - - # Existing GCP Monitoring metrics for GCE per Network - gce_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" - gce_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" - - get_pgg_data(metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instances_usage, gce_instances_limit, LIMIT_INSTANCES_PPG) + get_pgg_data( + metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], + GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC, + LIMIT_INSTANCES_PPG) return 'Function executed successfully' + def create_client(): ''' Creates the monitoring API client, that will be used to create, read and update custom metrics. @@ -99,29 +106,36 @@ def create_client(): client = monitoring_v3.MetricServiceClient() now = time.time() seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval( - { - "end_time": {"seconds": seconds, "nanos": nanos}, - "start_time": {"seconds": (seconds - 86400), "nanos": nanos}, + nanos = int((now - seconds) * 10**9) + interval = monitoring_v3.TimeInterval({ + "end_time": { + "seconds": seconds, + "nanos": nanos + }, + "start_time": { + "seconds": (seconds - 86400), + "nanos": nanos + }, }) return (client, interval) except Exception as e: raise Exception("Error occurred creating the client: {}".format(e)) + def create_metrics(): with open("metrics.yaml", 'r') as stream: try: - metrics_dict = yaml.safe_load(stream) + metrics_dict = yaml.safe_load(stream) - for metric_list in metrics_dict.values(): - for metric in metric_list.values(): - for sub_metric in metric.values(): - create_metric(sub_metric["name"], sub_metric["description"]) + for metric_list in metrics_dict.values(): + for metric in metric_list.values(): + for sub_metric in metric.values(): + create_metric(sub_metric["name"], sub_metric["description"]) - return metrics_dict + return metrics_dict except yaml.YAMLError as exc: - print(exc) + print(exc) + def create_metric(metric_name, description): ''' @@ -141,15 +155,17 @@ def create_metric(metric_name, description): for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): types.append(desc.type) - if metric_link not in types: # If the metric doesn't exist yet, then we create it + if metric_link not in types: # If the metric doesn't exist yet, then we create it descriptor = ga_metric.MetricDescriptor() descriptor.type = f"custom.googleapis.com/{metric_name}" descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE descriptor.description = description - descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, metric_descriptor=descriptor) + descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, + metric_descriptor=descriptor) print("Created {}.".format(descriptor.name)) + def get_gce_instances_data(metrics_dict): ''' Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. @@ -166,17 +182,27 @@ def get_gce_instances_data(metrics_dict): for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_usage = get_quota_current_usage(f"projects/{project}", metric_instances_usage) - current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) + current_quota_usage = get_quota_current_usage(f"projects/{project}", + metric_instances_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", + metric_instances_limit) current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_INSTANCES) - write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["instance_per_network"]["usage"]["name"], net['network name']) - write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["limit"]["name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["instance_per_network"]["utilization"]["name"], net['network name']) + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, + LIMIT_INSTANCES) + write_data_to_metric( + project, net['usage'], metrics_dict["metrics_per_network"] + ["instance_per_network"]["usage"]["name"], net['network name']) + write_data_to_metric( + project, net['limit'], metrics_dict["metrics_per_network"] + ["instance_per_network"]["limit"]["name"], net['network name']) + write_data_to_metric( + project, net['usage'] / net['limit'], + metrics_dict["metrics_per_network"]["instance_per_network"] + ["utilization"]["name"], net['network name']) print(f"Wrote number of instances to metric for projects/{project}") @@ -191,19 +217,38 @@ def get_vpc_peering_data(metrics_dict): None ''' for project in MONITORED_PROJECTS_LIST: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, LIMIT_VPC_PEER) + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data( + project, LIMIT_VPC_PEER) for peering in active_vpc_peerings: - write_data_to_metric(project, peering['active_peerings'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["usage"]["name"], peering['network_name']) - write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["limit"]["name"], peering['network_name']) - write_data_to_metric(project, peering['active_peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"]["utilization"]["name"], peering['network_name']) - print("Wrote number of active VPC peerings to custom metric for project:", project) + write_data_to_metric( + project, peering['active_peerings'], + metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] + ["usage"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['network_limit'], metrics_dict["metrics_per_network"] + ["vpc_peering_active_per_network"]["limit"]["name"], + peering['network_name']) + write_data_to_metric( + project, peering['active_peerings'] / peering['network_limit'], + metrics_dict["metrics_per_network"]["vpc_peering_active_per_network"] + ["utilization"]["name"], peering['network_name']) + print("Wrote number of active VPC peerings to custom metric for project:", + project) for peering in vpc_peerings: - write_data_to_metric(project, peering['peerings'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["usage"]["name"], peering['network_name']) - write_data_to_metric(project, peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["limit"]["name"], peering['network_name']) - write_data_to_metric(project, peering['peerings'] / peering['network_limit'], metrics_dict["metrics_per_network"]["vpc_peering_per_network"]["utilization"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['peerings'], metrics_dict["metrics_per_network"] + ["vpc_peering_per_network"]["usage"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['network_limit'], metrics_dict["metrics_per_network"] + ["vpc_peering_per_network"]["limit"]["name"], peering['network_name']) + write_data_to_metric( + project, peering['peerings'] / peering['network_limit'], + metrics_dict["metrics_per_network"]["vpc_peering_per_network"] + ["utilization"]["name"], peering['network_name']) print("Wrote number of VPC peerings to custom metric for project:", project) + def gather_vpc_peerings_data(project_id, limit_list): ''' Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. @@ -215,7 +260,7 @@ def gather_vpc_peerings_data(project_id, limit_list): active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering. peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering. ''' - active_peerings_dict = [] + active_peerings_dict = [] peerings_dict = [] request = service.networks().list(project=project_id) response = request.execute() @@ -224,22 +269,33 @@ def gather_vpc_peerings_data(project_id, limit_list): if 'peerings' in network: STATE = network['peerings'][0]['state'] if STATE == "ACTIVE": - active_peerings_count = len(network['peerings']) + active_peerings_count = len(network['peerings']) else: active_peerings_count = 0 - peerings_count = len(network['peerings']) + peerings_count = len(network['peerings']) else: peerings_count = 0 active_peerings_count = 0 - active_d = {'project_id': project_id,'network_name':network['name'],'active_peerings':active_peerings_count,'network_limit': get_limit(network['name'], limit_list)} + active_d = { + 'project_id': project_id, + 'network_name': network['name'], + 'active_peerings': active_peerings_count, + 'network_limit': get_limit(network['name'], limit_list) + } active_peerings_dict.append(active_d) - d = {'project_id': project_id,'network_name':network['name'],'peerings':peerings_count,'network_limit': get_limit(network['name'], limit_list)} + d = { + 'project_id': project_id, + 'network_name': network['name'], + 'peerings': peerings_count, + 'network_limit': get_limit(network['name'], limit_list) + } peerings_dict.append(d) return active_peerings_dict, peerings_dict + def get_limit(network_name, limit_list): ''' Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. @@ -258,6 +314,7 @@ def get_limit(network_name, limit_list): else: return 0 + def get_l4_forwarding_rules_data(metrics_dict): ''' Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. @@ -274,19 +331,33 @@ def get_l4_forwarding_rules_data(metrics_dict): for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_usage = get_quota_current_usage(f"projects/{project}", l4_forwarding_rules_usage) - current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) + current_quota_usage = get_quota_current_usage(f"projects/{project}", + l4_forwarding_rules_usage) + current_quota_limit = get_quota_current_limit(f"projects/{project}", + l4_forwarding_rules_limit) current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, LIMIT_L4) - write_data_to_metric(project, net['usage'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name']) - write_data_to_metric(project, net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name']) - write_data_to_metric(project, net['usage']/ net['limit'], metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"]["utilization"]["name"], net['network name']) + set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, + LIMIT_L4) + write_data_to_metric( + project, net['usage'], metrics_dict["metrics_per_network"] + ["l4_forwarding_rules_per_network"]["usage"]["name"], + net['network name']) + write_data_to_metric( + project, net['limit'], metrics_dict["metrics_per_network"] + ["l4_forwarding_rules_per_network"]["limit"]["name"], + net['network name']) + write_data_to_metric( + project, net['usage'] / net['limit'], + metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"] + ["utilization"]["name"], net['network name']) + + print( + f"Wrote number of L4 forwarding rules to metric for projects/{project}") - print(f"Wrote number of L4 forwarding rules to metric for projects/{project}") def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): ''' @@ -307,34 +378,49 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) # peerings is a list of dictionary (one for each peered network) and contains: # project_id, network_name, network_id - + # For each network in this GCP project for network_dict in network_dict_list: - current_quota_usage = get_quota_current_usage(f"projects/{project}", usage_metric) - current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) + current_quota_usage = get_quota_current_usage(f"projects/{project}", + usage_metric) + current_quota_limit = get_quota_current_limit(f"projects/{project}", + limit_metric) current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) - usage, limit = get_usage_limit(network_dict, current_quota_usage_view, current_quota_limit_view, limit_ppg) + usage, limit = get_usage_limit(network_dict, current_quota_usage_view, + current_quota_limit_view, limit_ppg) # Here we add usage and limit to the network dictionary network_dict["usage"] = usage network_dict["limit"] = limit # For every peered network, get usage and limits for peered_network in network_dict['peerings']: - peering_project_usage = customize_quota_view(get_quota_current_usage(f"projects/{peered_network['project_id']}", usage_metric)) - peering_project_limit = customize_quota_view(get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) + peering_project_usage = customize_quota_view( + get_quota_current_usage(f"projects/{peered_network['project_id']}", + usage_metric)) + peering_project_limit = customize_quota_view( + get_quota_current_limit(f"projects/{peered_network['project_id']}", + limit_metric)) - usage, limit = get_usage_limit(peered_network, peering_project_usage, peering_project_limit, limit_ppg) + usage, limit = get_usage_limit(peered_network, peering_project_usage, + peering_project_limit, limit_ppg) # Here we add usage and limit to the peered network dictionary peered_network["usage"] = usage peered_network["limit"] = limit - count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_ppg) - print(f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}") + count_effective_limit(project, network_dict, metric_dict["usage"]["name"], + metric_dict["limit"]["name"], + metric_dict["utilization"]["name"], limit_ppg) + print( + f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}" + ) -def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, limit_ppg): + +def count_effective_limit(project_id, network_dict, usage_metric_name, + limit_metric_name, utilization_metric_name, + limit_ppg): ''' Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit @@ -359,12 +445,15 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met peering_group_usage += peered_network['usage'] # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) - limit_step1 = max(network_dict['limit'], get_limit(network_dict['network_name'], limit_ppg)) + limit_step1 = max(network_dict['limit'], + get_limit(network_dict['network_name'], limit_ppg)) # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network limit_step2 = [] for peered_network in network_dict['peerings']: - limit_step2.append(max(peered_network['limit'], get_limit(peered_network['network_name'], limit_ppg))) + limit_step2.append( + max(peered_network['limit'], + get_limit(peered_network['network_name'], limit_ppg))) # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 limit_step3 = min(limit_step2) @@ -373,9 +462,13 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, limit_met effective_limit = max(limit_step1, limit_step3) utilization = peering_group_usage / effective_limit - write_data_to_metric(project_id, peering_group_usage, usage_metric_name, network_dict['network_name']) - write_data_to_metric(project_id, effective_limit, limit_metric_name, network_dict['network_name']) - write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) + write_data_to_metric(project_id, peering_group_usage, usage_metric_name, + network_dict['network_name']) + write_data_to_metric(project_id, effective_limit, limit_metric_name, + network_dict['network_name']) + write_data_to_metric(project_id, utilization, utilization_metric_name, + network_dict['network_name']) + def get_networks(project_id): ''' @@ -393,10 +486,11 @@ def get_networks(project_id): for network in response['items']: NETWORK = network['name'] ID = network['id'] - d = {'project_id':project_id,'network name':NETWORK,'network id':ID} + d = {'project_id': project_id, 'network name': NETWORK, 'network id': ID} network_dict.append(d) return network_dict + def gather_peering_data(project_id): ''' Returns a dictionary of all peerings for all networks in a project. @@ -412,20 +506,36 @@ def gather_peering_data(project_id): network_list = [] if 'items' in response: for network in response['items']: - net = {'project_id':project_id,'network_name':network['name'],'network_id':network['id'], 'peerings':[]} + net = { + 'project_id': project_id, + 'network_name': network['name'], + 'network_id': network['id'], + 'peerings': [] + } if 'peerings' in network: STATE = network['peerings'][0]['state'] if STATE == "ACTIVE": - for peered_network in network['peerings']: # "projects/{project_name}/global/networks/{network_name}" - start = peered_network['network'].find("projects/") + len('projects/') + for peered_network in network[ + 'peerings']: # "projects/{project_name}/global/networks/{network_name}" + start = peered_network['network'].find("projects/") + len( + 'projects/') end = peered_network['network'].find("/global") peered_project = peered_network['network'][start:end] - peered_network_name = peered_network['network'].split("networks/")[1] - peered_net = {'project_id': peered_project, 'network_name':peered_network_name, 'network_id': get_network_id(peered_project, peered_network_name)} + peered_network_name = peered_network['network'].split( + "networks/")[1] + peered_net = { + 'project_id': + peered_project, + 'network_name': + peered_network_name, + 'network_id': + get_network_id(peered_project, peered_network_name) + } net["peerings"].append(peered_net) network_list.append(net) return network_list + def get_network_id(project_id, network_name): ''' Returns the network_id for a specific project / network name. @@ -446,12 +556,13 @@ def get_network_id(project_id, network_name): if network['name'] == network_name: network_id = network['id'] break - + if network_id == 0: print(f"Error: network_id not found for {network_name} in {project_id}") - + return network_id + def get_quota_current_usage(project_link, metric_name): ''' Retrieves quota usage for a specific metric. @@ -464,15 +575,17 @@ def get_quota_current_usage(project_link, metric_name): ''' client, interval = create_client() - results = client.list_time_series(request={ - "name": project_link, - "filter": f'metric.type = "{metric_name}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) + results = client.list_time_series( + request={ + "name": project_link, + "filter": f'metric.type = "{metric_name}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) results_list = list(results) return (results_list) + def get_quota_current_limit(project_link, metric_name): ''' Retrieves limit for a specific metric. @@ -485,15 +598,17 @@ def get_quota_current_limit(project_link, metric_name): ''' client, interval = create_client() - results = client.list_time_series(request={ - "name": project_link, - "filter": f'metric.type = "{metric_name}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) + results = client.list_time_series( + request={ + "name": project_link, + "filter": f'metric.type = "{metric_name}"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL + }) results_list = list(results) return results_list + def customize_quota_view(quota_results): ''' Customize the quota output for an easier parsable output. @@ -513,6 +628,7 @@ def customize_quota_view(quota_results): quotaViewList.append(quotaViewJson) return quotaViewList + def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): ''' Updates the network dictionary with quota usage and limit values. @@ -526,9 +642,10 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): None ''' if quota_usage: - for net in quota_usage: - if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network_dict['usage'] = net['value'] # set network usage in dictionary + for net in quota_usage: + if net['network_id'] == network_dict[ + 'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network_dict['usage'] = net['value'] # set network usage in dictionary break else: network_dict['usage'] = 0 # if network does not appear in GCP quotas @@ -537,19 +654,28 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): if quota_limit: for net in quota_limit: - if net['network_id'] == network_dict['network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network_dict['limit'] = net['value'] # set network limit in dictionary + if net['network_id'] == network_dict[ + 'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same + network_dict['limit'] = net['value'] # set network limit in dictionary break else: - if network_dict['network name'] in limit_list: # if network limit is in the environmental variables - network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) + if network_dict[ + 'network name'] in limit_list: # if network limit is in the environmental variables + network_dict['limit'] = int( + limit_list[limit_list.index(network_dict['network name']) + 1]) else: - network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) # set default value + network_dict['limit'] = int( + limit_list[limit_list.index('default_value') + + 1]) # set default value else: # if quotas does not appear in GCP quotas if network_dict['network name'] in limit_list: - network_dict['limit'] = int(limit_list[limit_list.index(network_dict['network name']) + 1]) # ["default", 100, "networkname", 200] + network_dict['limit'] = int( + limit_list[limit_list.index(network_dict['network name']) + + 1]) # ["default", 100, "networkname", 200] else: - network_dict['limit'] = int(limit_list[limit_list.index('default_value') + 1]) + network_dict['limit'] = int(limit_list[limit_list.index('default_value') + + 1]) + def get_usage_limit(network, quota_usage, quota_limit, limit_list): ''' @@ -568,30 +694,37 @@ def get_usage_limit(network, quota_usage, quota_limit, limit_list): limit = 0 if quota_usage: - for net in quota_usage: - if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - usage = net['value'] # set network usage in dictionary + for net in quota_usage: + if net['network_id'] == network[ + 'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + usage = net['value'] # set network usage in dictionary break if quota_limit: for net in quota_limit: - if net['network_id'] == network['network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - limit = net['value'] # set network limit in dictionary + if net['network_id'] == network[ + 'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same + limit = net['value'] # set network limit in dictionary break else: - if network['network_name'] in limit_list: # if network limit is in the environmental variables - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) + if network[ + 'network_name'] in limit_list: # if network limit is in the environmental variables + limit = int(limit_list[limit_list.index(network['network_name']) + 1]) else: - limit = int(limit_list[limit_list.index('default_value') + 1]) # set default value + limit = int(limit_list[limit_list.index('default_value') + + 1]) # set default value else: # if quotas does not appear in GCP quotas if network['network_name'] in limit_list: - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) # ["default", 100, "networkname", 200] + limit = int(limit_list[limit_list.index(network['network_name']) + + 1]) # ["default", 100, "networkname", 200] else: - limit = int(limit_list[limit_list.index('default_value') + 1]) + limit = int(limit_list[limit_list.index('default_value') + 1]) return usage, limit -def write_data_to_metric(monitored_project_id, value, metric_name, network_name): + +def write_data_to_metric(monitored_project_id, value, metric_name, + network_name): ''' Writes data to Cloud Monitoring custom metrics. @@ -608,15 +741,24 @@ def write_data_to_metric(monitored_project_id, value, metric_name, network_name) series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" - series.resource.type = "global" + series.resource.type = "global" series.metric.labels["network_name"] = network_name series.metric.labels["project"] = monitored_project_id now = time.time() seconds = int(now) - nanos = int((now - seconds) * 10 ** 9) - interval = monitoring_v3.TimeInterval({"end_time": {"seconds": seconds, "nanos": nanos}}) - point = monitoring_v3.Point({"interval": interval, "value": {"double_value": value}}) + nanos = int((now - seconds) * 10**9) + interval = monitoring_v3.TimeInterval( + {"end_time": { + "seconds": seconds, + "nanos": nanos + }}) + point = monitoring_v3.Point({ + "interval": interval, + "value": { + "double_value": value + } + }) series.points = [point] - client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) \ No newline at end of file + client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 4a797900..f3ab73bc 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -155,18 +155,6 @@ module "cloud-function" { } } -# TODO: How to secure Cloud Function invokation? Not member = "allUsers" but specific Scheduler service Account? -# Maybe "service-YOUR_PROJECT_NUMBER@gcp-sa-cloudscheduler.iam.gserviceaccount.com"? - -resource "google_cloudfunctions_function_iam_member" "invoker" { - project = module.project-monitoring.project_id - region = var.region - cloud_function = module.cloud-function.function_name - - role = "roles/cloudfunctions.invoker" - member = "allUsers" -} - ################################################ # Cloud Monitoring Dashboard creation # ################################################ From 18a59285d2a37c654bf5f2c8edc9ae1a68a2dc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 17 Mar 2022 20:35:33 +0100 Subject: [PATCH 09/23] Sorting imports and constant variables --- .../network-dashboard/cloud-function/main.py | 38 +++++++++---------- .../network-dashboard/main.tf | 32 ++++++++-------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index ccbc05e7..8f9c5458 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -14,12 +14,12 @@ # limitations under the License. # -import time import os +import time import yaml +from google.api import metric_pb2 as ga_metric from google.cloud import monitoring_v3 from googleapiclient import discovery -from google.api import metric_pb2 as ga_metric # list of projects from which function will get quotas information MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",") @@ -29,24 +29,24 @@ MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}" service = discovery.build('compute', 'v1') # DEFAULT LIMITS: -LIMIT_VPC_PEER = os.environ.get("LIMIT_VPC_PEER").split(",") -LIMIT_L4 = os.environ.get("LIMIT_L4").split(",") -LIMIT_L7 = os.environ.get("LIMIT_L7").split(",") LIMIT_INSTANCES = os.environ.get("LIMIT_INSTANCES").split(",") LIMIT_INSTANCES_PPG = os.environ.get("LIMIT_INSTANCES_PPG").split(",") -LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",") +LIMIT_L4 = os.environ.get("LIMIT_L4").split(",") LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",") +LIMIT_L7 = os.environ.get("LIMIT_L7").split(",") LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",") +LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",") +LIMIT_VPC_PEER = os.environ.get("LIMIT_VPC_PEER").split(",") # Existing GCP metrics per network -L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" -L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" -L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" -L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" -SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" -SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" -GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage" GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit" +GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage" +L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" +L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" +L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" +L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" +SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" +SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" def main(event, context): @@ -64,8 +64,13 @@ def main(event, context): # Per Network metrics get_gce_instances_data(metrics_dict) - get_vpc_peering_data(metrics_dict) get_l4_forwarding_rules_data(metrics_dict) + get_vpc_peering_data(metrics_dict) + + get_pgg_data( + metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], + GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC, + LIMIT_INSTANCES_PPG) get_pgg_data( metrics_dict["metrics_per_peering_group"] @@ -84,11 +89,6 @@ def main(event, context): ["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC, SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS) - get_pgg_data( - metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], - GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC, - LIMIT_INSTANCES_PPG) - return 'Function executed successfully' diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index f3ab73bc..bdc94f77 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -18,22 +18,22 @@ locals { project_id_list = toset(var.monitored_projects_list) projects = join(",", local.project_id_list) - limit_subnets_list = tolist(var.limit_subnets) - limit_subnets = join(",", local.limit_subnets_list) - limit_instances_list = tolist(var.limit_instances) limit_instances = join(",", local.limit_instances_list) - limit_instances_ppg_list = tolist(var.limit_instances_ppg) + limit_instances_list = tolist(var.limit_instances) limit_instances_ppg = join(",", local.limit_instances_ppg_list) - limit_vpc_peer_list = tolist(var.limit_vpc_peer) - limit_vpc_peer = join(",", local.limit_vpc_peer_list) - limit_l4_list = tolist(var.limit_l4) + limit_instances_ppg_list = tolist(var.limit_instances_ppg) limit_l4 = join(",", local.limit_l4_list) - limit_l7_list = tolist(var.limit_l7) - limit_l7 = join(",", local.limit_l7_list) - limit_l4_ppg_list = tolist(var.limit_l4_ppg) + limit_l4_list = tolist(var.limit_l4) limit_l4_ppg = join(",", local.limit_l4_ppg_list) - limit_l7_ppg_list = tolist(var.limit_l7_ppg) + limit_l4_ppg_list = tolist(var.limit_l4_ppg) + limit_l7 = join(",", local.limit_l7_list) + limit_l7_list = tolist(var.limit_l7) limit_l7_ppg = join(",", local.limit_l7_ppg_list) + limit_l7_ppg_list = tolist(var.limit_l7_ppg) + limit_subnets = join(",", local.limit_subnets_list) + limit_subnets_list = tolist(var.limit_subnets) + limit_vpc_peer = join(",", local.limit_vpc_peer_list) + limit_vpc_peer_list = tolist(var.limit_vpc_peer) } ################################################ @@ -134,16 +134,16 @@ module "cloud-function" { } environment_variables = { - MONITORED_PROJECTS_LIST = local.projects - MONITORING_PROJECT_ID = module.project-monitoring.project_id - LIMIT_SUBNETS = local.limit_subnets LIMIT_INSTANCES = local.limit_instances LIMIT_INSTANCES_PPG = local.limit_instances_ppg - LIMIT_VPC_PEER = local.limit_vpc_peer LIMIT_L4 = local.limit_l4 - LIMIT_L7 = local.limit_l7 LIMIT_L4_PPG = local.limit_l4_ppg + LIMIT_L7 = local.limit_l7 LIMIT_L7_PPG = local.limit_l7_ppg + LIMIT_SUBNETS = local.limit_subnets + LIMIT_VPC_PEER = local.limit_vpc_peer + MONITORED_PROJECTS_LIST = local.projects + MONITORING_PROJECT_ID = module.project-monitoring.project_id } service_account = module.service-account-function.email From 73269eeed51e47b1295e23eba3bb30058397955f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Fri, 18 Mar 2022 10:05:51 +0100 Subject: [PATCH 10/23] Removing unless variables, using local modules. --- .../network-dashboard/cloud-function/main.py | 3 +- .../network-dashboard/main.tf | 15 +++------ .../network-dashboard/tests/test.tf | 31 ++++++++++--------- .../network-dashboard/tests/variables.tf | 1 - .../network-dashboard/variables.tf | 7 ----- 5 files changed, 24 insertions(+), 33 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 8f9c5458..0b1e3dff 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -155,7 +155,8 @@ def create_metric(metric_name, description): for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): types.append(desc.type) - if metric_link not in types: # If the metric doesn't exist yet, then we create it + # If the metric doesn't exist yet, then we create it + if metric_link not in types: descriptor = ga_metric.MetricDescriptor() descriptor.type = f"custom.googleapis.com/{metric_name}" descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index bdc94f77..152bb360 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -41,7 +41,7 @@ locals { ################################################ module "project-monitoring" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../modules/project" name = "monitoring" parent = "organizations/${var.organization_id}" prefix = var.prefix @@ -54,7 +54,7 @@ module "project-monitoring" { ################################################ module "service-account-function" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/iam-service-account?ref=v14.0.0" + source = "../../../modules/iam-service-account" project_id = module.project-monitoring.project_id name = "sa-dash" generate_key = false @@ -81,7 +81,7 @@ module "service-account-function" { ################################################ module "pubsub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/pubsub?ref=v14.0.0" + source = "../../../modules/pubsub" project_id = module.project-monitoring.project_id name = "network-dashboard-pubsub" subscriptions = { @@ -104,16 +104,11 @@ resource "google_cloud_scheduler_job" "job" { } } -# Random ID to re-deploy the Cloud Function with every Terraform run -resource "random_pet" "random" { - length = 1 -} - module "cloud-function" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/cloud-function?ref=v14.0.0" + source = "../../../modules/cloud-function" project_id = module.project-monitoring.project_id name = "network-dashboard-cloud-function" - bucket_name = "network-dashboard-bucket-${random_pet.random.id}" + bucket_name = "network-dashboard-bucket" bucket_config = { location = var.region lifecycle_delete_age = null diff --git a/examples/cloud-operations/network-dashboard/tests/test.tf b/examples/cloud-operations/network-dashboard/tests/test.tf index ad03d9de..9e467726 100644 --- a/examples/cloud-operations/network-dashboard/tests/test.tf +++ b/examples/cloud-operations/network-dashboard/tests/test.tf @@ -22,7 +22,7 @@ resource "google_folder" "test-net-dash" { ##### Creating host projects, VPCs, service projects ##### module "project-hub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../../modules/project" name = "test-host-hub" parent = google_folder.test-net-dash.name prefix = var.prefix @@ -36,7 +36,7 @@ module "project-hub" { } module "vpc-hub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0" + source = "../../../../modules/net-vpc" project_id = module.project-hub.project_id name = "vpc-hub" subnets = [ @@ -50,7 +50,7 @@ module "vpc-hub" { } module "project-svc-hub" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../../modules/project" parent = google_folder.test-net-dash.name billing_account = var.billing_account prefix = var.prefix @@ -60,11 +60,12 @@ module "project-svc-hub" { shared_vpc_service_config = { attach = true host_project = module.project-hub.project_id + service_identity_iam = {} } } module "project-prod" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../../modules/project" name = "test-host-prod" parent = google_folder.test-net-dash.name prefix = var.prefix @@ -78,7 +79,7 @@ module "project-prod" { } module "vpc-prod" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0" + source = "../../../../modules/net-vpc" project_id = module.project-prod.project_id name = "vpc-prod" subnets = [ @@ -92,7 +93,7 @@ module "vpc-prod" { } module "project-svc-prod" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../../modules/project" parent = google_folder.test-net-dash.name billing_account = var.billing_account prefix = var.prefix @@ -102,11 +103,12 @@ module "project-svc-prod" { shared_vpc_service_config = { attach = true host_project = module.project-prod.project_id + service_identity_iam = {} } } module "project-dev" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../../modules/project" name = "test-host-dev" parent = google_folder.test-net-dash.name prefix = var.prefix @@ -120,7 +122,7 @@ module "project-dev" { } module "vpc-dev" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0" + source = "../../../../modules/net-vpc" project_id = module.project-dev.project_id name = "vpc-dev" subnets = [ @@ -134,7 +136,7 @@ module "vpc-dev" { } module "project-svc-dev" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/project?ref=v14.0.0" + source = "../../../../modules/project" parent = google_folder.test-net-dash.name billing_account = var.billing_account prefix = var.prefix @@ -144,32 +146,33 @@ module "project-svc-dev" { shared_vpc_service_config = { attach = true host_project = module.project-dev.project_id + service_identity_iam = {} } } ##### Creating VPC peerings ##### module "hub-to-prod-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" + source = "../../../../modules/net-vpc-peering" local_network = module.vpc-hub.self_link peer_network = module.vpc-prod.self_link } module "prod-to-hub-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" + source = "../../../../modules/net-vpc-peering" local_network = module.vpc-prod.self_link peer_network = module.vpc-hub.self_link depends_on = [module.hub-to-prod-peering] } module "hub-to-dev-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" + source = "../../../../modules/net-vpc-peering" local_network = module.vpc-hub.self_link peer_network = module.vpc-dev.self_link } module "dev-to-hub-peering" { - source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/net-vpc?ref=v14.0.0-peering" + source = "../../../../modules/net-vpc-peering" local_network = module.vpc-dev.self_link peer_network = module.vpc-hub.self_link depends_on = [module.hub-to-dev-peering] @@ -205,7 +208,7 @@ resource "google_compute_instance" "test-vm-prod2" { machine_type = "f1-micro" zone = var.zone - tags = ["${var.region}"] + tags = [var.region] boot_disk { initialize_params { diff --git a/examples/cloud-operations/network-dashboard/tests/variables.tf b/examples/cloud-operations/network-dashboard/tests/variables.tf index c1833b7e..5339712c 100644 --- a/examples/cloud-operations/network-dashboard/tests/variables.tf +++ b/examples/cloud-operations/network-dashboard/tests/variables.tf @@ -24,7 +24,6 @@ variable "billing_account" { variable "prefix" { description = "Customer name to use as prefix for resources' naming" - default = "net-dash" } variable "project_vm_services" { diff --git a/examples/cloud-operations/network-dashboard/variables.tf b/examples/cloud-operations/network-dashboard/variables.tf index b3b0c23e..7170a513 100644 --- a/examples/cloud-operations/network-dashboard/variables.tf +++ b/examples/cloud-operations/network-dashboard/variables.tf @@ -24,13 +24,6 @@ variable "billing_account" { variable "prefix" { description = "Customer name to use as prefix for resources' naming" - default = "net-dash" -} - -# Not used for now as I am creating the monitoring project in my main.tf file -variable "monitoring_project_id" { - type = string - description = "ID of the monitoring project, where the Cloud Function and dashboards will be deployed." } # TODO: support folder instead of a list of projects? From 44b32400e6061728df334aeaf13a53cea4487cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Fri, 18 Mar 2022 10:10:08 +0100 Subject: [PATCH 11/23] formatting test.tf --- .../cloud-operations/network-dashboard/tests/test.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/tests/test.tf b/examples/cloud-operations/network-dashboard/tests/test.tf index 9e467726..40bf89be 100644 --- a/examples/cloud-operations/network-dashboard/tests/test.tf +++ b/examples/cloud-operations/network-dashboard/tests/test.tf @@ -58,8 +58,8 @@ module "project-svc-hub" { services = var.project_vm_services shared_vpc_service_config = { - attach = true - host_project = module.project-hub.project_id + attach = true + host_project = module.project-hub.project_id service_identity_iam = {} } } @@ -101,8 +101,8 @@ module "project-svc-prod" { services = var.project_vm_services shared_vpc_service_config = { - attach = true - host_project = module.project-prod.project_id + attach = true + host_project = module.project-prod.project_id service_identity_iam = {} } } @@ -144,8 +144,8 @@ module "project-svc-dev" { services = var.project_vm_services shared_vpc_service_config = { - attach = true - host_project = module.project-dev.project_id + attach = true + host_project = module.project-dev.project_id service_identity_iam = {} } } From 5d7e25484d2552c4a554b3c59584c50e15f8ea7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Fri, 18 Mar 2022 10:43:16 +0100 Subject: [PATCH 12/23] Improving README.md and adding a picture --- .../network-dashboard/README.md | 7 ++++++- .../network-dashboard/metric.png | Bin 0 -> 143984 bytes .../network-dashboard/tests/test.tf | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 examples/cloud-operations/network-dashboard/metric.png diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md index 5ad3e638..126651ca 100644 --- a/examples/cloud-operations/network-dashboard/README.md +++ b/examples/cloud-operations/network-dashboard/README.md @@ -3,6 +3,10 @@ This repository provides an end-to-end solution to gather some GCP Networking quotas and limits (that cannot be seen in the GCP console today) and display them in a dashboard. The goal is to allow for better visibility of these limits, facilitating capacity planning and avoiding hitting these limits. +Here is an example of dashboard you can get with this solution: + + + ## Usage Clone this repository, then go through the following steps to create resources: @@ -17,8 +21,9 @@ Clone this repository, then go through the following steps to create resources: Once the resources are deployed, go to the following page to see the dashboard: https://console.cloud.google.com/monitoring/dashboards?project=. A dashboard called "quotas-utilization" should be created. -The Cloud Function runs every 5 minutes by default so you should start getting some data points after a few minutes. +The Cloud Function runs every 5 minutes by default so you should start getting some data points after a few minutes. You can change this frequency by modifying the "schedule_cron" variable in variables.tf. +Note that we are using Google defined metrics that are populated only once a day so you might need to wait up to one day for some metrics. Once done testing, you can clean up resources by running `terraform destroy`. diff --git a/examples/cloud-operations/network-dashboard/metric.png b/examples/cloud-operations/network-dashboard/metric.png new file mode 100644 index 0000000000000000000000000000000000000000..3a6ae4d1f759f5c828448a25aaebc72cd66e7c9c GIT binary patch literal 143984 zcmeFZbzD?U+W<^=Bg-NUB3**y(k+b$O9(8DbV)Z$$Wnr|AR!o}NJ=gxNP~38u5@>O z>+?QwKi_x1{`LL&p5O0qX6MYzH8a;WS07?^v{Z-*?h~M)p%JU8D(a!3;Q-LkF#7Ru zP&IgxF+emlA}0q01syd71tuL2S6c@s8#FZ4*f)l_;HSNST$9&HNs$yWp{c{!OrvqB zWX4$NAH(C5uqd9u@a4J^__*p{l6So-f5p+*i1Fn^Of%1`X8xED99so^uL=$yK6tm) za@c$n06uRTgZ1BNBD>KX0&x&|~nZ(_j(gnq>?k*|%qNaVhOj zokz?QDeY*?W}iDVxtdgB$DY+g+8#0|U>-a}6aE0o<)K8o#Qb2lek8<9dPCXCM-c|b z2%W%-!8~WlMYFWoX7%ta;c}X^WgvY9ow|oC2;0sA+kn8t3IQldQ@U84^k;n>jP`?DjfV+>vSN1gX#$_~5KM##pPU#8jh zsC;r(rj8LCi4b|Dl=CQ9bb4_rRm4LYxZ6^M;D2<)DZh?RQ|TBS1#b!3Q0 z?h^pnw;xBVKRpkMK=oqJu6?6~`9_5y(l8hcJwO%GF}BvJeij zP!{neaOaD`65p@0v`Dl5ZbIRU@pvV~7unt&eSl~yrn@`4l-n^RrhQQ`e!uITO4`Ib zHo;V&Jd7!de8-&?gm%_ZxFm!pVmL>8M$^CgS!$UxAG)96mNNXLUQ)@FK)Z# z^hd|!TW4N3+V2}-xU08G^p93G=J3RpcZW%7Nh5Qm-lY{Lm>5w__{@d;SjrAja8`NA zT(|~VWBM|GCbU7~mBl-oNT~B;4Gk49zTqKM1JNWF9FOGIAbaiz_+p@{B zEP1vsSbwt@^mF=U+hnH_+F%8e+xmiwzC4g@$wZ!TS6IPW{T8I{7_Szly$IiW*i1)f%$?UPL!dctX+t^_2))d8*ukhYS zK96LL6msMDCR-qnReU&-_6z&Wl0Yy{EUq3_pXnPqrRv07OBta0;5Bn^@|yN5<^xr~ zB9>8{QRz|q(Z_nY9}N|rc4%}wW7S}Nrcj{!LHARY;c9*N<9NIKLDtS+9#5y!DFBk} zUK=L)yx!_H{7%0n{feY3ZADw{owY7Nr;p=js!%F!YF$=V79dMHYgM=_>*(SKG0dyCd2`=?)~DHl!;hueSr3YzgSX4|>9Ltd6X_)c=H!3#Zz zp*hD)p;bBLz0$;&2}?)K39ezTpAx0sN}tuUOPvb`Mi34|Klvu5KLwe5Q;E(Odcv;E zZfKV|X!e#+V#r&qQKT`y5o2DdQPN9$qgF^DiY=mJNeJ-h<)c9XiQ^dmT7T1Ko@R>X z8~*|S`OC)3-AlDgq2nJXi5qo~1dEANe)L1iEf_2pEQ9H6AHk$F#GE!NHzAWz&7wdT zGHWJb4iPa6x3Q%{<{tv%0<&&!AZ5!6taIPn?Q=%F3VD_7V2x2*#AmdFjAo$g;r)Q_ z4^baLuBJv>IBOUYUR?wZ#}udPsp1JlLs=N_WNlk*-)R{A0N=1n)^q4QaX>+HT~O_f z?bT|YJTmWttX|325GpnV~Ui-A=>6@q7LSy264Z{r+ z7e7@$Cx2u$j0cGx#>GXeC;ZS#zbf=niKlrc_9=Lr))3dQ#j#5Gc1{AMXI_4FJ02X_xdq&U?sjg=&`U$VhB~8{g(1S4+FrHw$fe1J zzk4n>Gu7Lze>Y^j{dPfj>wl{*mt;})?&&Xn zQ{KR|<-971Rlnj8L{`6+?z<2A<7|i+(O2enO4iN;{(1B%&63|_*u{~u<;pI`ySlOE z4@^A#KW&WDJT)mw7-ezT5?*xJD)#=tV)Jb+Ch&C=pEfDOCO`M~JsFOfL(Ib;M*$`5 znlc~-yD01K^-L0>^ua&_GK2jwoDWKgI$_M>ScxRl^R2iJyTW}FK6M}b_|0%}o0e55blh2J>KEMq!bTU#ia({2b*YQF&`Zq$ z$cyWbv?V=nU8E`f5ZBX{)g^;fLvO|QLRyx?_n?K;&HNK) zhfVsP)2oNci=bf1(d}td&#tW}^Uv}=pYRr%B$yL6y*%{)eo`%3FUmI$Ynk@oKA3nr zOX*s-KtAUeEQ;LS3b;7C+#L#$_0_r7xq@u4uBuSA&fW#T`&oSgHc_2j?>R`^t2>%C zmrjxcEF08j~1(?q{PJKg0oJD&wrpQQdW#7ggf84IVL03EPtVl-9xb%#qD2r7d`DXva6z-@L=#w|34bK#O-)Cp=p8~YyCeho8RRYtLjMr; z_7FJafVITrByz+oy8TAwJd!E2v4Qw;&3B<0T#=PK^)iWs_{6umzB{u&@lf|kAa33?SO{$&o)}9=kF^C zb^q@3KTpipQE0fRUlgd@_Z`N+TH^rTVg9R((T{qECihf9O%3&YYUN>LLrVL zF9%hD@1|4G#e0=zPg!o)N?Dz%6#l`sr9`QeV#EWXd>*?p>1@Yx| z@nrp&t(31o8h~`d_xnc<`MccFBtr`hMMINCQ&W`F_eI}+i{BLV^Zd^6 zC+YPFi+zKAL$gMt`acrKPx!S%7%taa<)fC)|!+OAkj;|_t zcg0`fZGi-Y&viT~|Ii54^>rvboKo(88|5Ekhn3@V@OZ`d5wrafq7K-NyHK{2}8~nLqd#vm!Ur>)OmCP&%H*%H|02> zN`E_*f6-Ui2D|Jj^B3azfW2mDD7slJ80^u@_CtZ;Fv(rCk{z;K{%!Y(?sB_oMC7|l zMY^UaO%5(t0MVT8pGFmMz}h^-i6NZ2K<0(q;w$la@$ys_!^`RxvbMOGxOke6M->X- zMJE%l{!Dql2+_B%A9It_#HN`CUzmpEPxgHS0CWVwhUjb&WcZ_uH1CJGUFy>RL@gSm z=q%K#(Z>}bH{9uxOrfEzX6lo0j1huJwWNU9E_&54>*PQ6SG91%t3=2XKL9&bb@eGQ zUJmEdF4k1K9Sw-u-7m69e=gF6T}b6`yF>CPS_^YvlO2b*-rekZ{NSxiRRS0lfmX;9 ze>F%wCnOY|_j*b|xMs@oqMhZ>XkQ{f9}2mBiuVv~pAYS0WxV*FD>+UDNmc%di(2gCaqti=WK3KB5#m=A z#0-&RhCd_Q7lmvb*M6s}KSG>ALDZo2C;cO`|Bt->g^&LuuYVqQf&U|~f9V)KRxz2h3fJ!u{i)Npe;`{TLK{2Td> zf;c{j2OOne`nYV?-x%_d226GIp(L3hz&!APW{NfY0gAQ13qi4y%bwA!zZXQd=;&CH zpDS7#uTfeP|KxL@=gM5LpS%TBWYi)v_Jvdv`Opz@V>#foSTQJt+OWf@40D;_7Y0Yx?W7S-ag~2UkX30&buj{^Bj_r zV)P2b25`_iRmx4feB0?e{K22cnc`pGN&kQ;j79-XH_Wm9@ucZfzLxWq!^Dtl*4fkH zXz8y`wbQmXWi=Ch9Pb+*@^SF=XK=ny`1>Um$DwsA5Jj0^j;23)vfPf#NcV!k5`Ce! z^mAJPQH;}&2q|b^)?Yxh&aQ{b0G!`+-o?+C=s#X~k2*^A_D(cL4CuK0-I%KKt$jY< zL|aD4?S`vSKI;0rU7uF&lD98sKqKn?^VCgC z*f#Utle|SL+3HFV2w(i^z0)_0xWV9tcSjK>Azmew_|+>q`>ai`vMDn>R`p!15m^Kh z+&JS}8k}JWZ+npieJxH6(uaSM(qTqgp~$`8nrV*Ls2V5Neh9(P0tR@XZ-1Ru&=jwI^!`^~K~oZXR%5iY#-Lu5|ZRW-r`rB-5C zm8o8|-0Zlfqp;{x`swhhgG92FNxy&HV@y-@4S&eRk98EjtW=2l=aEu-e&Tx)E&}Q@ z`rwTBn3D7Jjhml7_bE|7*LOiF@m~(QY11`D=Y>5JG=09|Y@7p??kqko7^_u&vq)YvB)ZzSIaNYIoSu6jL z|3tN3X@+JK6`sv>=uj7hto21XdR9KP{bjTcQrN}n|}SeVMdW3rn9 z`Tt6E(@#P*(0y6*d{<&K1>c>fAa_w_)`d;YPlF%Z=vIEClx3k2Y#&nv6er{XxJ{Yl z?7z9to%7U&LI=;4uf19XUf%MHICW~?o?*Y>-Q?ZeMwwKC-4nr5 zzp(H~{f7b;o(A7+nWv+8%!vb&A^4YzzW94GYqOh8!}Pq{9k+SGji3=F@1`wYvubXjo)R-wmH+a4&a7`b%F;4>7zD zhrD$Sn4Rt84?c*4Zho-z1b0H%RNHsy(C<0} zwu!;Vg=6?J_L~g`FO;M1YqnL><#}@zg(weCTc>CekXXj?)`Ck0qNF<@Fx8aboZMpX z!wDBp27!b9>*Hl&&Y^l`(}0~CdPjvo>%oDVquCf}!dN>g`xBIoe2vwRFTM&B>Z4se z?`}e0j=s0benEF2tBl>%0cj#cx{bPi9Z~k{=!EW}__fnC3Rw5zlG+?41S+5hA`zLg zh|mvVKX5aUioip0AF6fFyem`0r1iF7S{-%g=3AeI2GhWUF27*6++D zAS*`4f7@X&t#|$-lx4DO^1L9E{cTVhgm!2=e z4Cxy9TCWeUS1%9?0ej8`eX2=~<<>Xh-?5z-d7|RlhI_$rp|YEtxzL!o;svf19nw;x zI*7=dQ+?I_<;?nKvYW7@d3$n^QpYf|F*9%zouqDclDJiQMOE8193L1!KeRLeZ)8$@ zAhnFI4V*kFFq(YVYzPK^C8td~<>G;byu+*blso=|r&`0WM{*&VobkHyv~YhlF1>9i z__X>u`C;@jO7txze!4Ql6@{oU26=J8i+nDQOJwHHtia3t4`l6mI9p@fSP7O38E<}p zh;8_)(+HU>6cq*2v%8P0BC!W+DuQ+#9boKKqxT(4ljXZXxLMt#c$C3+SBrT&*3DU% zBw|cQ-$b%I2ZUDe7AWViSOmzVo4&Ke35y;z|Ch0`AA%)7WZADQ+}76DNPx`o93*4R zO-8v?7KND(89E>p*eV>kkh{b5R-*TYE1B3mE$e9}<&&E@E&%}eZ3p9RQaIfF?s9U? zv=IM=9oRJUF_pl~p{dW!$s{ShAh3&uQ4t%9fo_n@&iA&RNo!n$&~`Pa7d!!%c8dED z8i8=)+JR$i|JHq4UXmvxe%tqz5>$XEgOV)H!B?}3h#96B>wD#-G3Na?6Baa@{WLh< zY1b+cNE*?a##R&@ey$Q3H~ww$W>;TM?Z;u6YY_LO955vQXty^u^NL1LbH!o>){5h^XgRrKk{yPUej51NS0^G`7C&{MC`s zW=+;crrD^!n8_Rbimqv$*~!$ zAqAx930UzE{J=o4T4wcCWK8R_*Bx(p>lDbb%Omz9o>)|%2s4Y5OEL%*i0-6MBu~r6 zcfnX18JN;m1Y0;Y_^pbWF0t!q!ZJB`&*o(N2?CLg*Y#8w9yK_!@QuJqm2H z5+x5_ovt&1K)yDj#Demy&zBY}x(x09_b!9^E>P&f*L25_W|9>n!4W+qn1AwHEW_B% zUws6_gy?i}d?=}Yy=n&R5J~<~WZ1kMMf-&-ciw$-`ObIvhzM#Ywt}GrYQ6_s6jOVq zaDkGrCT8bw>f+?4bblNN6Fu1~5(as>9nqa=Zk|aK*JjhsEc+uwHRcezq6Mom({)=O z^0JZtlra)Bw~QwJl78{XNoiap&Wd$GqthT2Hz)rNrKb*uWp)hcfipVW?UsRh4NP=S zC@rGFf{%LYHyYO)x60MEjfirA@paR-*+f5A3TAkC&O1oyuJVT|Rupr;YGG5+DP_pb zL!#*qRVrMfSJbnJU`E$W1uY>m@a}rBryRmE5w0vj4mebrUzdk5zYgk+?!jQ=0Y$o} zh`6;cpq!bW-~4kfXZUz3$pk-2@HMQx&R?w}qo59ypK0t3=t(44jy)#lyZ+T?U88gP zF@o_ztNSWNTUvZ}kaT#$#Puy<4LtSMqKM5-{G-C9u0Uyi|5F(!UBm?p8f{?g@Se_U zLgU@u0GyZ@5(1Qm1h!*S-y?eEq_W`C9S*N=Cm^?Hf$FPsl=e$Cu^E{8t>>wkzI9jpO)$3L8oON5m#i$vO^>B^67=O0; zbcQ>zqk(`F!C)0>C@D^aZ{8JP^HuNaIg*=&@+L~$6OEq$v1bomeFQKx^Rrp7X#e1s zJ6e(q8>>WKmrY_Lo>2ilL>#4i(x<(-v3K4;NY1FgU~$3PKoXkmhv!_qJ++xf>!vP; zaqlc&%7$dJn))#wks|%+ zahqJuca{RJv^uaDa!O#+S&rqIN1>k4Hx2m@o9+Yd)iO6Vnu%uZys!8g;d$uOdiA^( z>Pf)Q4Ddvrgp~qko4I(sN$$eZ8_o}-8pF`Z!-WkxsYyC9A-5)NCk{e2T4 za`u~4Wl*N_bS)&FADzWsB`)ef;=s=O7T$^uoJZuv@iBxIxV4(MzwVJ@1?&AOx6f z)H<-b7+hemfIb}a9O$J}O3nawqwD~}UbM^hgcq5D>hVxEMV>aM<4w!OlS?|iE(;AC zW0qbnVBy^C_kx7u?wJIZR}uzIjI^_50}Q<-%Jwv5-o zg(C={aiLbsf@8b@$zw;c1FY9^a~KWl-9~x@YI_(0Qjs8}IDMh$xgfl735C8V_t6Dk zK{7h_p)>Jev|ZU=oWL(LNa{R@Vqr6IR^V`i|Dkl0)a8@NHbp{)_2Yxor&_(Iqgq4_ z_hx$QEy+4%pO_1-JgW!_7Bk{_<=&zCea=FIYwN{YKnLxDj@en`!8Ud9GqAW&*_vU zCB5wRod_0u2xRYlpNRgNaO5|!9)8pg;+pM=XXKf}6^y7-;$Sjo5#>UiZQLHX4ot3g z0XS+9nfOy)N0!V&?roVonFAo9`fMQ}9TocEM?ON*i3%=W4ElVb&v|+rO{?k>E*8Cn z(W3!Iljz2h@vk*7wuc4q*n{|+cH@+ zuE!C~BWncbT-2{7{Aj>+?T(Zd3WiBdvY)%h7nu!m5)eT;8@$0h99?HMFg6~#kn4l> z@2rXajtCT#c58I?gsYab=8FBGI;bPd4Z`1m5GkNhA9c7NzWjE}JichTvDcfvJ?Y5* z_U*Wx|VMsf8vRD?w5W>r~G-7+&1r$vwp0e9!4|(eqE(v!p|#)iwKc&+{hz20INx zchI8+b4&cnX+G~?4*(%?mnRh=Z%;p*2KgGSHQ=Si5P-(^orJRE;BY5~`SxwvQvxVd z?fy5{kgHjT^N!&Ncj6Ap?q{%9KpTNWh*u{i6{c{vmWP8C#~Oy(GRCXM`1VxBM?_+# zNs?4BjncVn_`9xi+V&xlM1+MKcniZ&QmJs$@dG^=V50p`W;FD@a zD#DM@cY!64FxiNaBt}uKM;Xh{h(aa^monVmSKgr_V|h5Luus6^WtiZb(`SizO`l?w z^BE~Q_pSI2F|T8_16i7pjzNmGOm0g9{nr)?f?CJWgF;u}uG&bXE=7kJ;noV2ZEwpR{_|Cn2fB3Iz_gf)`sX6%i-KljyI zDvPgdBW6K*^X4#q%^X6Rz7=*ROC4g>^f&Kha?X6a+%$+P+{&Rxk5U&O) zp{>=!Fjs~?mgHvj?CLW{9sf3id$Dwh-IKGm>)oZog}Qxb@E8%)R+!qIZ4#3}?piA? zRubV!MxZ+Qe5yw}&8q8h${CDv6`mspEZ}8*!e1Osbqy2GK2Nt0=1|p?+oq6vMG&GP){d~7h?ZhV8f3WS6+;|@ z6zIk+_srifg+BiXctmzAd5QfEdoX_h?pZfz$(C1FWoU60Mv0xx=t(B|e2i~qO*DH^ zTOZu^ZuA+x>zM&i*%;#W>&4Y{r7jR|0s%fOD5Q$YHUOHTsvQsffG5W8?oQB|DFQ2h zZoXG#8L%<|Sj`GteEdcLRKs$kJJEg%S)#uzzZsZaIIT4RzBYywy?`crPfbE}UUVrj zZN)Oe4NUJDL&O}RZb_?fclgsLs;IuXgqLH*0+uy!yjcQFBW|r6C!ffg8hFNZ0mCI# zxGFbMbw>hq_l5zmVQLP3*UK8ffuPYDVQHf%f3AEL&JQ%E1Wau;iUK$TbED=(6Gj~D_6p7Zv zB1fvJWL}mDyd$M|1-Xg=ZtyE2072`(<*r47z4bM!s73?ep)sV*6>8_}L8x0uh0~H> zMY3wj4a@G$xFBEAs})tuFu>!!5Wnu2ZJ@&GI}rAql7?LO*LEkm4Td4oVe0gf^zdYi{cZ(%I)F2; zJ|u$-GYCz6#9rb(@+&1{GiHt=o|SPqHHKKYLaY3L(cvGIP~{$o_xut<_Ay7(jfOv; z`~H>XjV?C}BsGQerZK%*L1*_%MBjY^JS=MDW2C-1L>|M+3z%YuaCg z$Y5Z;C*LDw({x~UY#Leim!TX-81D_kM>k@~ocNg)Z~Poj=W9@1<`(4Fkh|S&q+4bX zj3Yb8?hUo%NNatpQ0Z5*dXs`bqCWtM1e-0gcTDtyae?_;B8&5rPFNcGba*)ngFD zHmzJ*RB=n9BXG6hgJLv?IlL;Mb)(XjxSPjF*AgjuGTVs{HxNJ0aNbfrmUBB zZsGKXlT-lZb^|CjkvQ;O{kiue0;|w2)fhRBQR*6j>z|{p5Of7W95JGf9qiSoMR)d4 zT*q4M2nL+~m#lp31Tv-hmB^RCbQbLHB==PRY&Zr^abcpge{P&~lm=09G9N3k4$z99 zXcauxMw3j+>0%)UyC>qX=6Ih<9Dw88Nhnu+^3=1T*ab|E@S zU)$f!xMaK^+;s~{PV?IFe$ z)o(62p)jvtX{8x@k;R0cA(a0Cbp3PY~AUlQdG&E{?RJ8%yTkZXdxh;qZoX>C2*8%RqUW8EB) z9YVM-adNBtcAaJdXX;MwC^CpJrk?rkCTMUfAU&IH_i0FSO$`yox+pIMc?BFp`F_*o zXCH?v=7Ywb6#5R`H4RrKiR66EOI{u_Xq#XbPGx>X7j2W(XI$PW3F$_hN;Hry>n4VgCVJ>!pPu$VfS|tw z>Pk`}gbL7CyRTgaO$yZN+Yaf;F@V~$@i*hK?m~L^*XxDs$UbDfr-g(a^ynm>i7H>% zc4}Aibqce1*i&}@k{FtbqRqAKlwW)2iV~CjFe=gXXFQ5tAE7&}R0e}`2n^8=tbRI7 z05?y3+D^;@Uk+fXU_rHvr?|YT$}udzHHJ?yUXkk~^D3YBlAjj+^|IRHwxcuuk%-5E3Nn${ zEQ4Yy;w17e?2ybVq)!+JqoS1}eWxDQ>tZ}ZC^i-WNLa$p#XUDC=yX~E1KZ3fg*i#f ztEI8^KTg`Vxrc3@6la@?ONN+`p!ga*AtAY|BmcV2`Z@$nC{m>W>ey^;v6Kl%`OJv` z=-Z~I)SC@+x3A_u&xg5I`ir$ppq$c)CWZmeAn1v~AT1Rxj)tGV6dD0A*_vY$FjGxm zZt@qqqHtLqC6XC=Jd=`eJMmX^QCuwOs^X*iu-{0p;+S|#@JR?qORv$Q$kE@`6UJLN z_ee%%o2i+2q5AEuz@XAXi*Qs|BYFo674>SX&i0bN;X9T(Y-Nx8?jvn!zU3b45@Wj6 za3i^JkRZJFbI3yl>r5UhhGC%R(N+>&eZlP})TNg=`{H{+d8vFeqmLV+=tDE-RvCBhNERpos zamlx(pjs@%BlPX8oQ|Dv`5PG$@2RGLp7A{caGP7?G8HCK!z6y$N1IPj>|oI6s@Gm1 zO)Gb2$_n3~EAE=veLkR2!}Lfd~d<%_v*NeXwR5sn)8L+$_`k*~in;T^kw zq$-78!P46yAVt1>D#^vv!8kD9kRQc*%!!T_d6%fOUDI_cyZsv(^q*|x0{TTsCV4m^xTlcIrRd|ZjSv~(e77S5_PWDSk^RG^uoev;KM{L27r z!H449sZUTjzqyJoe+kYDtYi76J4BH1RaKz$@#a716t+ID{uQfa)0#Ei$8U zV0(_88B3rDl5KA*L))NJ{<(RWNM%BYJ+#FPxFRw;%Gb%;xoCV3z=d!A7gxS5B|6Ls zlszF_vlFx`?N^F?*iyofOFuKlolFvoT&>Dwo=KF9xnp0U$?o^M69fS8kK@ote5~PZ ze8YTP`!v&PS($>D-gB8|LnwKF@yi0p*v;cE#|dfTUS2n15k1M69f~4m=~9Y4A$kJr zQun<}AJdj;wyJ}xHz>O7P;}=|kQ1_vuBNKSr_L&4$;EaFtFI;TFDIL$V(1mA0<}J+ z>@0XsG!7|p$JjiY(@}B#5c_Z6R}JB02JlK_mXZMch@D}d0nr22Ea%vyW+x$Md{!sV zl)xe1DBb$VEOWSp5uCOz(!DxzS*(?m|CZv#^ZXmgIM)HIF&r}>*Z-V}sEKWq?@i1g z2gHl}D>KZQt$^tXu;vTNEhC{txI6 zc{?;UJotSG3>RwYTp1~xpvGCEpw6dYAfEDVrYP&Hfq(By#AY?;4+f1Jd*A`k1W=b9 z)Z$3S;6$#Vl7|df1~goH{HPr|Uvnd0;90Q_UT9H8S0?qmC*k{S%FRkq9GUJ{G7udE zNS0Y&Y)zTVmm{eo!p4bIz6>SuYl(y;DtB2{M-XZ z;63AEM!bm^T@W)G-}8;fOE^Sj2T4|BHYI*Yph)Hz&^(l~`r$X&&?ly3{p5Ef{s|9A zdZ_jU6EUvF{#HLh9Faj`M(C#*hgaH^>%nVfraV(hAQ+3B-U8zh56mA0DM~#!;;o%x2{~i6WWFN9SOm>phW$tF2*t9MK;Ly` zEpw7K{GE~IXCD7H8fl%|An)~ECbx#=r?+%oWZ`6`%Rykz(ZGw(WoaS>5c9z=cb&#z z4nvu{E(aQ8y~={~JBQ~HYN1=QMIu_WXfOG4-*TD~R+{XC5`#X^DB)J8H`m>tWtfz_ zj4vxhQc#bS6PW$--52qE>{GLf=4#z3??X-)NlE@ORP^yj?-w(~pfe*U(T6tNu$GUz ziFOhiFFP^BCyge<<}Q81S9=AFnM?fRhw&m@5MU=4850&jR?w%tgT7V~Kn*dgUEE{P z=FELxmFioiZz^(l77en3N|{X}$9#@?mplKsS#5R!lKtcA2Nq&Ccv$e}=jw?G-_ ziT8#Eiop-)x7FRL>O*-bWxl>7nG{6WNLr1OR4riiAYH%cF(-fFj1035`jsC&IF?{* zZa-$%K^XCYXr>cmU9*Ca6_EAyuh~2Tv-tag+7A@O0ui2+oLUXh?CYE(xWBV9%4vIr z;P!;yR+crm9u$Pl<`LS`DvVOr1$b%q+|7N&}RS52FGM**{QzPukT3ZHB#b{dLD&XZm)cAfHIt%skFSuN4`f|}f{!Y1`w_P=zS)IO!@ zjDgmRy*bzAgWS`2FEhG&uW9@sH-T}JB5u_7D|&84@{iI;)!~m@g~1}ze?6jf~(n&U*|`Z<$dvmJRC)C%AH4qMO!$)@6&-S#)1{-OkB$JoYn%~f+1cv#Bq zK_(M}QC3o8t;e{oij6*SlCD5l`ISy=c0R?5LM{CeX1Spvj0|5<(2`m3Jn=sVv5k(o z?MQs&Op!;&m*m$mk4~PaN(VexF@Epsm==AWk;<%3Qj3r!Qiop_(cy1+V zSpY1W8vc9Ls@~UV@xPM8qQ!J^Dy^0e>m6t& zUyAr}q-n?rKYUFFa5lp?NC~}v{>nZ*txV#LyCr)po2m|6^Q5e`n6(t<4n0sV&$Sik@pV?#~&m zDT81`iPggrl9yG0z4~ITt;bUIItk*1`achAgnM98IJ!fmfHDhDC5KD3MoDFx<@?fv z1s7{JjTmgZjzaOP=qh?EJE6cBFAkn1*e^0&!UkFX?ut|Sb7p0Ky@%w+Q^Jv&wUwdn zNpX^JQo^<0xtpvD5A+qpZ`y|^>9%G{rkQvw$P&FtMwtlj5O8=+9*i~01khp|oU)0R zTx`Tr5yU+~{C&@=IhshwIanKLyf*3D)yUgG9OW@satgK~`^ytO)pT z^0%Iq=&PDmlP!|Pznr#7PdM3Mwtg?ak|kd+v_i@QQF>9LHJPUszuHJo&LmF#JE?Gp z)z^K#e&rRnF`g_8J%mS^q{Ei^+1&3uzU$mn$C4b6Cr4X>+P++q6yAbn)g}f_rYIcMpaclK9NATEPj2Wna*U86 zvW~$923&Mh&gIhPH9?d8A1F~pMuVqBn&H!VjQCjG<8B3ySv1vLa7l@rYcy&r+uY0J z=g#Non1B{*2}=4VrSIE-5Vr*EIH5%rzoiXsOG0^7_Mbi?c!c|HzP+ZMW7##hEbpJ( z?-k5Fl3G!S3`I0SX_L9DmelEM1J~)`9a(ihRJ9tDxcpT(#?kLH-obcrdQe{Ro)M1W&HPI6&T=_dSeeygNiMW1!;{8}U-=lE zBgN`2P@ZB}4Y-Wx5nH$hR*RQdLG}!ahgV8vL*k`_rn_oQ%U{?Syp`+qWFFXP#o-tuyA7JO(70Gnh5fmm<@FHn~ zV8>?SX8nGh_=;$oR!%fX34U^Z;G%8&RO?>L6+$HD$JKVV_Mme){bPSi9!S`wC1l5d z$24z}1FIyx^?KbeScg;;BPPVOW7A)>pEuY%Yf^7h7aSV?mro2W%E^{)bCG0NIJvsM z!K``==;&3U0rokb-%Z;*?+y#fp__qR_C9U#YXj)dYe$jKv5MHD&QhdwB>s?C=3bfdK&vgmd*&3 zM=|;|`<^(EQ1FGNeUC3}xthj7Cpv1#9F_A~RL46@Nz-QeA%+5v*f(l5HcpO_9AhNk zXhNGjbZDHxy# zLxZ$*=Md70f^-R}Al*opN`o{ABHi8auG!n#-tPN*pMSpNJC5)9hvQ;qu63<-uCv#9 z{w~v(&w2uJ1EiQlx{7tRtf=J8JdJ(x&JYp2V6O<4NOFE(C3B`B>nu@7v0GS{S7e;W z-86GUZ+hQS0b9DaxVPHe8PzWdHOH!|g}gNnh^QOY8)uN6;OlUNy&6ig2wbVtuX*l) z2Pwj{!AyBWwQLHJ#6C8v?!i7#dQDGoqSX0lSjuzFlHrNBbK|}QZQDL7(Y2_T|8T)r zndHXEmYp}l>8x!?e_tJ+kEm@&ZTRtIW3i73GMKZ%Wbmqb`PRXWQxZEtq6x*3d9LGG zEpb(Zuso69WNb;dzj8f#vnx~}ciURjCZRTOd`tKRfiBwmV>8a>**&fM3-Nl|p<6AW z$)HVgjd6@3lG>i+;f(NMz4wW&CH&%TE%83|Wr|}W%So>s4+_8=7`tZiRSvI;f?mR@ss7w|PNFR{ zzJc5t&z;4|mWb2$iw6RfPNXst6}KHwgyGU2%qzP+S|cR_!z=hR)$^KChm5 zS2v9PX-&r&?g1;qgUCdgQUwnsNz%s4NqfQj?Z@?_W_`3F+6ORu?$Ns|<{w32CFRZy zPVWv^DvfCa38X{%dbo5``UB`6fmYy`-Ou-LB+YZpUiP=|zgcVjQOPv3o=7c00T%f} zsQqfEv#-Q4s*2?5X9EfMrA|W50W5-pc*l6F;&BPuYlO z=QmTa!QqsG{GM70B+G#%wI5%*Yu^m0M<-|hg%3#QLeHc&pb8=8_2RyuvppxKj3wEA z_i3r$HVAZRZiVtYK-JXrUF(A)Oqfrc!jEx;K5LOvUcAKK))L_#sozGbUCtch$V@;9 zP@VKG#i;pla|cLYUAO_Wya78>a2jz~x&!O_fU0>Tm2D=)ZXg}cW+sqtC?O6h8_l|GEM{0rM5I@eJl$g`|(2gCsa6&uQ122;IUtY-UVh4v}N7idKLj0z%Lv z;oq?Us?Dl{=qgG5b4ayInXB#;a_AnJSw2}k_i=CXa%4Z#B7xPCInTDbWImTJj6=8e zk}j;hqFy618ufGb{_abk3FnEW&K`bFUu){qHi;n%iF2kx7EJtET%TLCVNp}^4~ zrTi5ChGhJO7EEe}ZQ#sXthRh&yhmqyo0YN77iC@{6&vfs4)+~mu8M;VkS*>7%kMU%!;%%u#UgFdNn`_mrXZ|t8FZ>dajIi zIc2QPv117CZ=9e75yTmyJV~Q)vI(mTae>($)M@$f*;}_yQ1xn;@$yjWha%k!Js0q- zwzGZnk!WrUM|$JnBoR-_-qk+sM69u$bdf#^9x>~%)G1uC@!@jen3hTmQ&F7QTRg3& zl3wriWu&F>r1talyFA{?MS8HFvleYSICE8pvG~rbO=f6dhsRCNKU^C->6k|y=Y~HD){}`^8&z}bxUj|R>*jGWSWa8EF~{)CW;((`u=Th) zH)G2yEPKmdQknHmd0h6k+Uo<_CKzoXEE-+1l79G=FLCk3FUD6K-S*j;PJu|DB2)_! zFtH!JUCk1i^3L%s3vM4uxK3+6u9@c?WHrL*yx4*YCM{gId6qDQTS_PsSsKa(mZ8g^ zKb>x7hqR}@^XoMpCW%c7AbTL%D}vO zx4IJIvr(q%bHz3IkawhU6FV?-IJ)kw7AItWJf1Ks76$BTm?*-4D!f&ic1cA9UOOll zNjm9(SQkQ2SU4Udd#Or@4n1)KeN?UVhEYFvT@d|8bSq7I4}+iYe3UFAdN zX5%7^n`l#fn_gE(%v|R@b9`-$gzc?6;B>mmv}&V*|K#=6t`WpVdzg~1x*8vW&;_^ zd_65;a}z=qp|KO*BS>WA>!}y}flOhr?V)pWH{MdT4$=d1l}0m>m46FPgQ%(Gn+?X49T~rG5d(0qJM1h;s`dI zr6!I|QQy1)J7T%d@&oA38?_;VdV1Rb%LD0SRyTKJ$E7?E3DOgRHr?+X^~ zO6jH~1fKp2Bt`87vaB!ls5}kmLo5C2;=(Q%l9vbck;+DQjy5{!-19RKezFJB(q~O0 z;aTb$DvzGszQYEZQyypnzf_jOwnC(z-Xb#(E}v$2|Cf`BU&v-eh}J_}IDtV6QDmc#rGI#Wx)01y(Su z3B<3J_P=PdT4@yUil6pKH7*V=9%yy2DxXbs{QyuO1TI`CvRLp$834s|XpB&(w$o(6 zTxu{11mZ#yHkLnI!>acWCGhKz*#KjvD}aF+-#ZW!F%E`e3poj3VA%861is7d0-v=&R;}7+ zy_3=Vh2#X_J1()KNurO>?rLlDk6Jer0KBobsEnbo^LV2GMr4{SX;1nE0GRlg=(_U4 z|K!!ZLoa>Kl0iMg{po1OcgXWC?d^XgoE9TsplK7P3!~1n2}oY~-u$=drqOW0orNa1 zNhIZSbrq4~Y*5}|7%sBIifTWrTv~7BKLns9iP$%KUbE-bqLaNbKf83CuLrccP+bNeDQotcw49EeJ6*MhK&!5h)!+_W zysCIzXIf}JQFgh$QijH75BNAs(8igT5uJnuifxLgC_C{hLR8RylW zYv=`C7xJh0r)&P*4I$tMJ?~kYso&wN6@!a?9HP&}SuCEa^)Tdur#Ml5b$31*}C#_u27e19`Xt}%r zfF?v+@a_cAI%sH`P-!kKZXEsn!o=#>!zR=c=C%P_<0$l4&=E4W+`013Ys$>E#{Ou_zzV_9kDJ!ew82_P&$e zeyOupFQloS6+2qn00FGWhg44#LY`usL7yZo;jg7RzbLm@F#lLyS*LKEbyUf(BxapK zw3WwZoau#&YG}C+)#xL|=rBHC2L1ou#84EFhR;pANEzsR)?hgR!A91+0~*g>$J_U7 zYSJ}t0{j=-di#!ad@y}tS^%}+mL5@U1o}2OP;d>#YNfIkkl9Zt90pXU<@219;Y%Ym z;9MrGqEott22_wnM$X|kN?D#8pU}Dbnj1{eCM)XHy6 zHHex6vNqq;+7r6x#1>mK?aU?%SW_ zF6@Fnx15N2JPcNJ@okrk4%Cy%;p`+h>Csch*F}at%)MQ8R6@#=yFVz1gjP^ zSV(RWp5aC!-A#pB0B}N5LQ94PjqgbbSN})JaeypBw;4l|_BL2tXadWa@qq~$jjqx) zYA5;%-j7axSZcVDi~&cdR!$AnAAneHrCbWKlOU)Bq29+H1+EAD08#_pvzeHG%LXW1 z6)C>l1WfuFk&rp2;}oJcvgW9}Gw4V=MOdv>&GAmb@WVILI<}EEJ1@@;7O=uMXF>RO zS<5X@l7W`gOrbX31qCwW@T>!mAxvM#{?qLtf#Hv^UOFgS=39pjeLP6WgPiaacA(wa z9P;+|QQej#0~*O3G)eB7bl%n7cXxa#^;MngvJ)XF;3&`&@z^%rx%{@YRhI+)?3rMJ z!Z2+8wtJAav{$TccwwJtKE4us+wzNH{N*xcABbZ!jESHK2IKZok*UkOUbm)Dmp4wN zi`J6)gRF zo$Uv<%yjv6fM)a34Rm$hNLzcN!1+S5*;wpw$ec&kSvuk2QaeVonDtRlbahlc8n(_b zhB|aubhT>pQMU#3R`Xh?Q-N5Et`NcJ^vACah3o4x#g6B{R&~o#Kf7CV@M+p6+*K=w zlSy`>cGOz{a)zk`v6rC)M<~P30z?|KFq;J1Tj*vXYav4@mB}01tG(mg^E5FkIFQLh z*KKB0aMF+L0}7>Oi87d)DcPE!8F{s>jAyrG++~rRI8YhkwjZ(y!4PsoobHs?h#(_U zI_#Vp2Jh72ai2xQWD=GIR(W7^QRwr&GAb3-YaO2D1NSAyQfVg0&wUPB z%wM7UTG?8w)7vEM|9zRsmc|jBcnXURqj`9~ps05M&N>sGt`mm@3aM@ zTDVM?at|A@F@!AY56?$><d^D55P@WCwJ6UP$P z$5YP#Vn*svW~5moG8S`+H}@|-hyY551KU-2?K-8@0$s>N_N12$4bfigmn?h;Z2S5H zsRM?pEN(YFUgqMD@`m8JI*PmRdK9KbU9u9dv;_p)v8srBXW+^(6$ECz-fT>LVBg9R z@RhL{mm1bAwv!+J41kFVPiu+ir@N)B03wiu(Z?ci!wU`D zTskBc>aV|Xj)c_Q*XBHfZ;>dhbFvV4WG^GU^Jylh%ybU#;nTs6!-Y3m3jk&w(aVMT z+ZqT_J+F%*eXir1i6NunN3UM8vDczC2;GD@jq9^te1vurxsP0vilrs^0{fszOjYM6 z9|=&5t}@+%7ue=cIHB6JKLFUP&adsmxP1Y-@$NAKc~Xr-PE(<_mb9)*`R5pXtdS`- zY#kxSp;)O(#JJ-ioosvoi-ecUZ4#nhn?j2#iL2Pg7P60Wusf^Jo=%@GScX; zVh}QRcs`g@o-;3uGXi}Dd&!{)$%Iw+RabR+&>%;$i8<+KBcm}HFM&IJsfUK7SQUvC z793bB8lQw~jC<2QvD-+RbhtswDD3G5e%h%4Wthb28`zbhmO_s|y9wX+SGTr_HAHUH zE()B1!!B{xt^n^^O7?oyOYX9oc_phGlYV{T>0x!r8lWvLDIE=T5H~mrfXzi~5NXz~ zT6*J6o@+TQ`I?g8U0>Zb{O!YT(BVQG2&1%>Ft2*46Y|jJHWsF(l8B)yT3V z>{^)OaRYZA4eACE2H}8XR;!E>UFxdV^{$pV#R?Wo>PG6S@fwsXiz8YuspQ@u+QvlC z2-`+s3C#F>5$wS8e7D1MiWZ$z?_g1~CyI6XJBTs45 z1e=w6S|(mBj3(FcYO0@=erZ-B__&V^lgpdRv2Dgp^UDH$v<1Lj@Wx7w#EY_xeI63La^? zkxZqnPZh-3i?-{N6-2p3kPN3Trr0hqr$Uc2vPP+d@GDR8ANRb;Q*FXi+v+NH=uT#1 z&uhl?x7y={_&~bPt~J*Zxe`pn`Zc+{Ot22ddG++OSn83-#CY%t0m9C7BgPFZG@mTO z#`sT+RM@i1Y;M5ir_Z2=5{uMUC66s*O=9%}^lA^w;ACHT|8KrIehAADKyEr|c znIy_Ca3cR#HAg-3F>$k}aIX-B8ac@WFGoR7*t<#+C^0ttYr)$y<9o1t66P?$FM+(T z$?UyZTqu~~&wGgo76M6W$fm^HQ^V7fzMKZ~;X?lA^VBmI$@D!DdYPJ_ssnzD=|-M5 znmq$N5_=xt5E|@*@HG)#$rb>tWD>zy^w2upQhDT)u*mRRnAN;ax>TTCvaNM<<6B>i zN7iv6OTH1(2_XzO`IU5iUCeZqyOaf)zS4gIT1dVfnde9EBktQOc}t=Ye4z0LJ^%5R zvb99=g29?`8;MlnL0{%f8B+{Em!of^e**tJcPQ^}SyE27N&!8P=)U$#`1j+ocImHB zDyHN4EqNvxc7lpL@hFU|OVv4_l9)6=qQv)D4uI z6Bh=IXzIxdi(ZdviE%$#Z^?}Dx+}E5ef`u=Hv##WyRDE;dv1yXssZuZQf^aWygB@C zr{ywxL`5L4JT4oqqJ8(&5+cAeaFkv$y}Y^sz&)yot_P{4=SikRZZal7_0Nar)nWx# zGqur^)8~amr3Po=Gm>cFppQ%Afc3+ON|n_w~$KYaXpcp4PyeAZoJVW!Ymkd~*= zSnz1SE6sy>R!tF*BSJZ-smzpaGn$VpBNcuUC(tD_)nxyAkW=HH18Cr5n2wz3Kt9>a$?tv)5iweiC8%?0?L zqm;hauoq)p@G$tAbyCl?CUi=6e{E=ye?Zug-BOF*YCrZkC__?j47+Dz36ujjRETKn zdgEgHVPTT0F%)NVQ8`5Hbsj9{Cb{PvGcA*m5b^1u^*asqu7XlzYf;9tsx11~7hdkn z`3%cq`WQ8$k2sP-1K|=ejKvgFO;n`WLf!13PNk9DkMN-R8d|(LY9}Q=VQqBt(r9w@f=?l zo%<|<{{UaV22&ao6#-UgR)?|ZdDG7G%zLe6$Pt)uay}sPLNv!C*yV$yD;WK}04q?| z@pK4+8#MI&OoA)u?Q?RU=830cR>|kc6rsP00x((cDim4Xb7A2ap=zXkc&6%9z zAt{s%i)V$F;GL@6-Q4W@AuG2vlCP*yyMu6;1N7EAaP|c=yr|l3mwAdx{!Fi5XOfO( z>KWjL*y2iw1Pj#6@a4X@crDS#N-?btWo4(y+0S_N3DYwFtDSLa~B4f zsu?F}+}7&Y;9aW#`_eAaf+P;r%%W8mXX6BXujmmVV1We_!mjTXw4clu!4H^-d3(i4 z&Vi%q^gH#fDp9zbq4~1nj7^_FUr#@`p1()I9h~h?O|7mZP?*BO_4GD508SfktF*Fd zSFJ)v|Cn1l+feska`xg+z3L^O$_DI$q5@Jj$BZ;q_9GPygsv$gLF=xv$Fb4lL;$hO z{q8GwzbNoQ+lz5D?8;`2PeE?+h4!2$z)$Sm`^Z?~ROCpVUt6Xx-Yg!9_QFmWg-&ZL z@dY&zsopO{o_kW9=poz*Q|g}EZY=d)d?X#9B82?a+9MW5uRg&jZ6wxCiU-*J!Oyh^c@!jf;He*qqCK7^9Tv~R^&{;9}*ZOt< zNiE$wiT7p^iNs522S>k&*GB;Mr3rG0MAO%Bw+|z=gVZFdl240*$)y8fpG-X4`39nx z!iBFeDL8=2HFDb+$|%6Q%gM?;gd`MsAo-jwUORXtAkgyaeEQRZoHhm^)dbKbSv!LV z_?R%YD-2E%lCNc7kAf`HU(lYqV|YF640#cpuPAjVU%z>K$;xet+O~r+-U0;dOeyn= zvs(%KLKNL3?iOXw{+u0GyoO5W66>2cB?qqCy#{3ir+!Fqu)w=Hfx?qy{|g;fmf9jJ zc)}83%a)%sq0)w0BWKXOqU6(y7;3~4XPjJ^a1ro}dS}wa{J(1ZVFA~$fZIg9^>es) zxv5=~Q&^_3tTDk_`WeQ0G%NcsmHSSeblqpNJH_v_PJzbKAyi7iK@Y%vH2F?I1ZXX8hj*9hs|=8)_ou(K13uC>l_lca@`TJB^-r=@Cb=M90+4>$?(G=PBmtR z4fBfm47SH8)IX{D*VphMw^VM`0p4k#xtq!j6KeKv^#W1)TjKZLf_{TNf!Em$Puo3l`LSGxkBE=Y_w1 z$V|FosxUb4P+2{>{I-m-ms0n_#cdV7f^ZOSKc%&D@8?E{pfTb;d=Z%E<>fzlf`Xw& zWjUytNnr3meFND%hGn8ubn~I-GC7)2RN-#IrvR;YZ1i~re2WR4{;UNkn_!dg+q#C! z@3cR)O(fR5l$92CX$0tuENqMBXMGCUDYa#Y#uj2__o8R(vZJ85g78s-*9XH3y56fK zgmW0Cxx}hj0j(Hn5+q3R3z#H~>CUf7AWJP;Qf}h6BnEcm#y`uGckFD8aOkw^QIs&B`AhjXI3EK zP02yrv?^{VE^O&|_4=_g8r(($QT)Yc32MI1x8Od`nn=299fU?|!vsskgOMo9m-p&2`%pAL zLc{+mw&Sq)@ll;m$>vr2S_H^If0WX@U`%(Qm;_%4yB+g1o5^cEiRLK%?rUsUCvF|h zUGIG3$*(&n-O7x@2noM47TL(FDkCp2`Y=>P?*nAu!ceELy|RATGZz_gY@hMUAL(l3 zB3EApl*=D#n3r=pwez=8G(U^FT||-V(QKA2Nq5;y+mvOyhlmk#FRj`rg^q>@&Br-Z zzK30l*c4(LldngMx1e7&Pd6n!0!zE7GAw!cDz1e1Rh`C_T{G9$t&Qt~HG|K0xVeTq zk;50mn}wR3bWtX&jjP!5~yANgd6Cw z6I!k_>)tCk-l>1<4lc$~DDPV)S~3Y7G>i^dL!9hsh@bv(maNN&Xbb`(k3(Z)yHWgNzb@I~XLTEjr?!XjxVx?)0cojo6((13oV3swvo=+qvVPZgL|8_OJspCL{a{QTFponOw9 zSlL5t&yIrgc%)bp(gxkGw3-Zn)ZskpMqFwZH$SI{o-PT@K^*F>4bj1JDl+NldgyEs zflm3qjPdIwf*q9(AfcLeZ9?9h*n5U~0X~J-CyzwMHaB&f!=DrWvF?i}0IV0Y#fO|1 zmmr<1u6Gbs^Z-h4@>F@|!vk-@kGttZV*Xq*hPqXfuGz;99Hq0*DPE%*crb@?3sswN z5qah&ejq-3UsWTiLYeAjx@w8qpTZt^B?x-9Ox;Mxh@r9F5PSl6A2UAi9tl4@de%H` zj78Rn0~2sa-Epc?rg!<5)WtSP+dTR z*C&(Y%q)o};mwMLg(QjGNB%L^5RV4qRABNYw!d!msui!e&~t%}HzXka5YSkkL|i<} z6BiG{S%!Fwo1K;%1BZ&#{8;x7gzq*%RMo$ zbIMqMBGAvOm;aaFw8qLVi1Q;-|EWfPA^cn{Sm;33=pSo6eljK!Yxa+~{~YN7$fC?q zJ)^$yvsC2g?LUZ;oHs%9^PXSH_x;mkGn86RdnEPao}bkFWhvy)c#J>Y{`2W}N}vVC z?3N|#FZC~fxUBcF{dvzHW=#M8``7jTFBAO-6HT1vJJMM@+}<813`>=8cv2nUw{2U#hHQDR?A4J{+-_QU4^U4qcV~!S7MOnb27xVr!-)Y^C2%qi9HPL<)-YplrM z&xkWRyysc&-WGqA=VX23@_KzRSMy;0RI20Zcz)eM8Yr*f50dchzqhi9j}J@7PokZUiwDwN!&h$hrpYNL z@QYN~FHm*IwaqBE zToW|TO51V6?;NF$i>C1USM@qaSHq-CRZ*+)qe1gRr75rD-B5e~vpI)ulNjhBJRVz9 zPU}scjk%uN!LZ`-*10I!H^cnC$cKwIop=9*uo`%n7nNe8kZ7ewDYa?&pklpgrE-b% zL8W~tw|;X*1mfgq-{wPfO%xJ6jt|CJbhNCd9>U;L)RL$_ta6pgQ$ z?JSjndbKG>&prL6lEt?+2c^{{w2Ln+1KGmg)~sigO+O#e+hq3=JSo#x^AaC&`Ovf7a5XoM4JFW2~2W2OndtWF;=|(&sj%2-hb)AVa(dA&iY$0}$ zQGx7BnSOXsJO9`P_Y?4mMT}l5k+_^3>p^3ppyS<<%oY97xs$KzosB9U2fc%w5MqzDcdN}`&0k;GsT}tD z^kq>;h}oa;e&)(1Ss@8Is421#-WepbS(CtNOc^>cfqX7MTyZ^JwN;oZsTn01l|LP^ z?c~>hYh+b=xc!vp->$lR&^IaB10h@$g6G|r*7ueea9SjAmTKC%FB&jmR3G}DhHv^R8mQnS}rGSV0+M*}s>u67xlrR3Ct4QM&KYuPSu=kIE}rkln%Jw0Y`9sbxX zkL<1495YXuAD9v_PBP3x@H@=yW$&ry0Uk1)DwH8HI_OYrHQ87?9&NUp~A^RhaFtY3r72I!Ke_C`?+PYH3+# zNu+sD&`m6$<$C)_Bu;PM2g21M%fCd8V~|FDu)SE?HfvQiKU22%IW~kWEx4}!(Lf;AFQ(?7*Y*m!)Xg; zq9LVhL%4p09il98ALix-7o&}vv8l7^5`^q89m%^Kbv1-H-yXU~czn1UtRH^$T0gC* z`HN1tVfTBTqZ{`qCwJ~!K7Nz?j{j}VWX9fTWI#*i?l2Du8`4nJ^J7kF3)YbQV~djU zOB{PuZE-_nzQ}Ot6Hp*-Y^DKYC)c6-C_Jn8KA?hq(_iHqdnb?K!oS)B1!F(k%XSDc zbGLQscW=e{hnWz6hQDT&(J{p8O))m^a=2W!v~juT085UP-MnV(N~PWdU!9|k>>`JH z=b?MpOmf-xA2rrv5Q#?lAiXhgG7!2>b+LF$)<=7IHecTBC>~bt`)^3^&t7LrT5rinV=&CHYE%o!{3-1s}!wbcxY zu?*u=?8?0-3RAB#ldcB~&^_w%wy(V5U5i84SX#6BE{4a^QtJiB_$UwL%C#tssuwa* zr*=(A`u$5orY>m~rQ;QrV!=?3#{=4)BKOHAiw4bmm8NFJ4zCMu4C>pgSapP3lnz}B zI~XO&H>x8vESzl%GJ9LjxpFk}Api8_&@hcEEPLecNiEn5JYVF|dN3tq^ekrzEhhqlmTv&J$Z#Dc}9Fl-mzYDyy zNJ6@^Z{xmYr3jL|q5oWLzov%kviX)XRI+Mw+|}k}^@JGcunyVFM=AvOjT6Xak3bc5 zI~3`0DO6+7Ff&2#ljxfa{-fg+*QuS6Q&OgaZ5rik>w6j^_e)cpcr^Cp8xbjcAJ)ph z@5oEw#NBqNWXbu}5wu{1^W0xt0q1VPaaQj{4h)*yWpP!%dQR?B_L5&8F@Jaap7>Rr z3kUm@+JQWUSPAEJPukUMpy0KW9uJRW*v%0ZemHa<7yJpSsqR~p8@e1v%FzYq>$f!P z2ArpoquCj%;BjAn8U3Z|^NDDNQXKTY<@S46`AQ&r9_%UY=6p7-dqL;<*o+&m4<+j` zfKq}JK=C>8>Atzl;@kKz_FGn;U1xq(J6#Uwy%{RuHRCqM{F0LyG0(6F&Dk5PfJ4?K zQ0?(ByQk9e!IU;|w)Q^dc3ubCIn#}~U&qsK%#|J8eM+gcQDk?Va&xv*rI(t7bgATH zht0-&7MYgR?Q@B(Q1|Mk4Y4Lrs+5c1z^)ITZ18^>NmjX3r2l@KMi5q4U#fhnT(zkz{$RGZMgTFq)M^B?{4x zqj9EU0@a&sI^s9EqVNib<$I+wlIxBevyj3m_V((}FBPHHl)zD{NmrCFDQqo(LuaBqWwj8-+hT+^ z>qau`FS_bVhX-SKV*X*_UU=2{M0@`#8mO`JGP-YJIREJ?J|7SK;8T2Azrf>Yo1*LDin3roi@N3cs?)-%Au90y1l;zAy2Q z9r&jk3fEYp`eS7G_s_fwz$Sjk*zo-}dRHRWOxp`G9;3D_p(f4;6^&kK7z>Gq{~QxF zo+v38`NoftuXqQMu{6hU>o2fBeE3kWFYBg#=kKZzO!|L7)xOtB_sXnw+hRq8j5=Z| zATyoe$Y5yG(UNgX7;T#C?n*Z>hjWp~OCy;oTXWq^*X-uHUN3JV{$5Ct9GWk3XYftp z(|_)nxB@QNBEJRvCENE;n7M`GMx2L>Ot}pg`d~tqS_e{P;wG0P<`9(^~*~CyE8cvx(}#9uKwosdP8KzF$9} zwrULKs-d9^qoWI^ysl<UIMFZnEqfxCrROq2$_^xV@oqs_M@1}w)?6{0!WSP@0_JuemX zZ}M}HreGw+30P@AICFEP(Ad0WJ|kad$9UTtqjz_BuTQw@ZwgT`gK6I!Gt~br8!)U2 zcsF#N&U53v)82ZE{=wF~X%40=JxT=;Q1P<%?>DmOg|c6mdz$mI)=th3pNvK6@jQ=F zV@$sORD0>?uh3&%L#{|db-cFHdA2eCEtqyc#sK6+^xdg-sMD;xNW1n_anbB^b_gd~>}&MJm8hJ2ok&d#$u`HOx54Q`D$EF=3qg z#j^1LyOxIIV4k7qBH7})n0O6^e3R8m;vB)DgmGwAY>qqo)n$o9w}HXY}@ zhhWYpI7i%_pkwor^=39LL_3zix96K^faU7aeZKvh_|j9tHCmbPO|qn*Pk2^+eZp(s z{e)S;$2557a4WM{1!`i%birxzWwe?q{OWxvUEQ7d&MV5^opy`kBG|@g4Uxd?0fsfT zz^-m-MIO?*uqHYk%*RCLG<02dwVI`M3`uYt^e!Fy&dg|lxY*Wi3jd_JxHmfZ96jW@ zN163xJ#XqnSmnxNtRa0#Zz-Z{tT82lGI5)sY@sn1gOIVCGA}lFH?D$9fV=1hESD&x;oW%;wdfd7LTQ1#lX?ZNns`Yh8U}VFG|n;>YMU4xD(|$sJutOT zsiVlTJp59D=Fxi%$9ba-PKX514)S*IzG+0wO2W%VJfTT#*n0h%DaD2 zKe+&_3=L7|gJsWBSL7q&vX$k!g7IyierQqhJ$bNW&)#Cno>!ea9RnpDk(eT`J0VZQ zMEJ3ZsG>3iKE$mLlBuN6hg?*7L5gwrK1WHrq_-0PFsq9}FC9mREIsiJcq};cFlUkr z32Yz|6-0$cZj`1KFKa$K>QK>2MEb4Lm918Hzqy-R(Yf~>Y`FxLT;PwcMMz!jxUJ z)_mWpNhbQibM3`Mth1j5r=v7DsE1U`^5=M%=rqo_v`*CJw^&v-&4Cvx9|ea0E+Rik zVp0IM?CJYEOOQgN&M-5}G6rq;<>De=_d=|=(xHcMcu7;N!$Tg%L?F@3LEsn;v z_gCu0i^R~Qjd4n`Gi4L2lW< zamtT>^U#4VzU)=PsPlr`)`-fmmI~jTg~nDfJ6cJXiY>;zV-I@KxQuY zvmP5c+Tyk>?D3cqq7fQ(bXBFBVyRvxYtEus{6sJY3UV0GqS z^cCj=4%02$Ro9i@biWc{K9k4&EqhCS=8g5a9U=l}j1N~#GhgzNUvrr2n$we(9dfxqThvVEOc8xsx6OYw*Hmev!>BM@4?0$k@x}>N@ zp40uks^ojX@p)Z5D`NFeA@t4w?l|vlb1bn=DhQ>vi)qN=gl|#6(=Z`USpvH8IVuQy ze;Y1xWTlL0yd7#_RBbGYl_9Q|E*aqB6ZC2b7q4H#X=6S!jm3=JZR2Ua)eh}XYX4Vj zkTP3n)DCi(m7L>Fo9%Xf7&~JEB_H}_QDLyJTzp)pwbRc2>{P~mDj zvz&a{6NmRn5K2zP#mmA+gZg#%Aei_#z+m0cG<^0XX;@b2Y$3ZOL)>)5MkKV72=#wlpU!OVP-#P3bv1%C`Y7Hzs z-OerQ+AO(mL4i_p?mFxC-=5S8q@#-9o3Z<`R8|$Rdf`l?ee%iLV*pS9y?qg`@zQZs zaAB=Me~v|zB;lIw~6;%L*RPDd$z8E*>-GvwGn4 z!1Qx=4&0?L%cW+D`kSQu>1;>59PH=x4Zen1m=YW~6npJ-C&cO*ib2c@&l}ft8+z_R zP`SzXOuT#=Wn2)>^Ug=>2n%ee|F~%YtzPE?M~-qUo$t}zDXyU2q!?6G?Z=Y72@V6G zp}*a|36Xj~N*#}@Z!6cEV}N&gPe=FIoVQLJNTAts4WSgaI&`y3!cUH0@H#(=eHc_s z5n~zD6!*g^&_#{n?)LksfH&~a7@|#@k`#j)sc=KslQAmn=1Cg{G+7)vWGSo4@ZGQH54XD+s6^)8YrdPrw- z_lJHz1)60q92x%mewm;wFGB*<&B}qFxd)|Xolr6TXHDxaJ_sc?0gO20%Anm;sJ=Mp zx}`l175E$yP;e8lXP8niJ2#_GB$3c+nZ7yb!6xL~(&|S$iul{DzRg%WEbVu?-_H%% zs-!^f@4Nn<=~7LO1*&K`Z#UCukzV-Z>r>P8I*>6-Q@>L?H&kf6)hUUr8`_dFBRU_C z?ms@n*vY=7Sm(Ri7Rz^8bZ@P}Y4+W&&#NaH25ICQlDKC4=B4fY+*pb1_c!8dNuJ4t4>_(QclG4F@6MxPJmCZfoy!A)s zH!g_sfWQOMhY_wx9!u;^f*)s9a2@}GM%j?kKlREl9OFF$d|eaEZi+!Vv^u{avZ+t( z2zrAUVZ$4FC6ZUE7kTm$>n~DUkSD>eTsYX7f7)-(I$P->`|soZt)8=idghg*Gg7Bf ziWR=vtP(>!RhXx{voyrbbwA!&87c~Yl5#g!+f>B=_8m=Te*<;q?ZpuRl-A5Ho{8<| zqy99J@y)_tzF2$9x3}Pwt3#mvkgvc#_zSvAbcZi`us*bln2v&3L@;{F^0yNC5)F{>0s? zZ_Xj&9BLZ`-BSNKUb6T*u;lHYJ^ybjy!8}Jgm1fK_Vqu1xMl+;E*h5jJs$FQKt-6A z6+L9X9o?CrloJ%&VfWhvRnLJ3JnL2|QTx6}-&El;Y9-Y~=HC-T*_h}knzq@*lgfmJ*q0w-*wPW+58@n9wq;C*Z#2}kR2lYA7qD2I?;KIJ3>I#q(6ai@p3kJ zH~s%rj;6LTh)f@3?CqidPd?($6?rRy!hw%JCFt)f?+vg4A8a?&zi)g8ihj;Gt}fp` z_&crTh2DMau-iu^ZvU(!Sv;z6P88pI%_nQL%kL|0?{gKu?hWi}I!P=xgxT5cCYWj#F5sKTa~G+Rf|JHT< z9?`d*3LG9Do*T&3F{ACD`oZQoD9(7;cOJzVCfq17O|$=ggpn-X3-~G4az^00rYZnL zxb&vRJO^y7>J327sK>c%-bhjcQ-7useAfX^49n#(P?z!&E`}x1`^Wpva ze0aWy@cVIc&ffdj``GJPYaRdh$^Ubv{tp?A_nd|l%J12WMK%ejg=GOmzMfJ0BXD)J zum>!o9B{V)#r^T8A%}wh-M&78MOQ^pjf-A%Pr-hj-DUl|;_q?u?FCq#1*xaE{#_D9 z?BM&pH7(|BlQ5|C4=JIG%iFI_daO z$H98W!*^7-D%h3XDt9w@oaf%aZex$nX-A8-w)uHy+)(B4QMY(zCgo+H+v(4-?d5ZH zHN46gOu^Bw+KglfOeY#l_u)ac)!*dn7AUxkMenzRYW;dyYerZfqkKdtKvu`D=lEZT z%-_YC18#64_f(SL0RK>uQw^H~08S*m%$rWoB8oI0_5&eNsFEDl($6pr(ifGL$ws$r zJ>WC6fHp*9edRl)o%+B=s|hgtSM{o*`A1u>F3)=arNDf)&A)io*o#X17;XOc2lvNp z7KXb2Yv9kSxKe$#_OZ|@=fo3H1+1$&+>d{|#XmGPiObZrx8HOLo}UV=QOgjmFRdG} zs-WVx(>qunFDuc=ty2vi5+Ybn*}-)?i$=XHGio*ierO8!(#*B_k9L9IY@V&3w-%h4 zt}n9wh7w7)@}1jfi``WrejkvOuj=WK5W7MGXW_wf^({*FLHcFQM^78n{y3(&c$Ufy z=BG&(kF|Jg0x3}_NNEZd$|?tii@Da%**S+q_f=F<`8l5Kol97M%lgbNxKfz$nia=% z!xxy+Mm*9rJT$LvZh3Xq0LK4+X5nGrJPecb`kO61Lht*&wGzM!#u><63K`ZhajD>x z4t1uNv<`u8EIwLAzVmOH18TIe^Wn;x$uWKm7vLJXpq+W9!`eyq1|Q*!Y>@!HXLdI# zhYHLjX7X9oO%ov8GYe@*jj})T{>hL{z(j&0w5@Mb_GI}r14FVx6r0>R3MH^28G8ym z?$qa`#kJ^Z<;sBrP8aBj79VyQoQvn43As<>m5C9!;ADiD!Xp(2rfuCo1Wf~w(P9w} zc~6Z$E1p2z{G9b@Wkps0&T9Ej%hWDlC6g3Dkx|y$?P0omFsj}ITpyER-#yNO`yz(* zOhKx{q{&?4&P3HVu&j@MNbCIohhZZcB1$ScZJirw1_W+u{&I82bw1p$f$m~H^ZLT- zIX*oElVCG1sRg*y-7`PZ9+aMVi@l#EhD*sO=9KO>lYuuhisy~h> z&a+6@wjgi2rrqvUD!@PdM8f)w7ZNb@tW+-I7)bwK{9dwWxl$~021XAB-y(Z1I5(P) zrz78lUlh|EeWqyWC z^|!EvFXolCoh{?cBG>qZTym?z%C(@)B3sLUIS*ASAOtVR*gVVIW_WBCZV61qTxBR? zCO3XS#d+SQn6j#)ai$LnElNF#3KlmK5pgESmDKO*ZdPXxdmo&vO=mBJU`m2qI zA#}5v7gM5tvVbtwRb3SPQ@kVUveRv8MHh4I;kXc`1F4%yXp*`?(ZBN|$%iZ+sPZHPC+Q;wHgJcj5-^#%4vE zP~9IKA~QEan}=qe1=Q?>)a<-=_65x^vlth489}2Z3x^hkwIwH;Cmrl6yanQFth3cdqCP*<#0E`+iySHK4LBF z)<#Gc-7(9VI|YX(IXk!3Qj&SpTOj(WUHp@Bac+P%VRLJEzc+r9)CCW40RCH2mRv$Z z<36;nZ~Q$?Zh~{_kg@b@J0#e&fLs`OSZTRS#wIX$`W%bP*=@-(i)J}BqCIXJhlKQD zZ3x>kaKk%r2b25BGx`{rb-k+Ys}#Hi_@830fVoOTj>L9iBMY+SeJv_Ow7)h^h& zpDr!zHh~@(^=znHjH|8~WiCE>u=CP8-2xH6C=mOzYA=sEJhT^E_XkC^Pb7y^k2DpS zW#^*Y22+b2qQuYsJlo=CE;&I<6Sr8Hzs@Fk!1s4L2mzOHDi*fB5l?jMv`VCE1?aIN zL1Qg3>J<>U>YHC?xW)r7oxHYEtU9~U?p=uG#dG?ptXos{>I$+G((uu5-QvjJ(M*+k z8NUQyT=UX7V8F!|d>@zkg@!QaIb+aV^T*N#)@uJ8p8I&o%I@QydP?}#Xu-qOMrRXEaVdVTNOSbM^EGeA1S*=ULH$dFD9%>wD6?w0p|rW z?JVe+jk(-LP&JtH1!q+2x{;pf0tFHJqnHs*%hv3uS+dvOYJ8}%T%T+pATo=>l z+pU?f^Ql{E*@?6MX3W|v8Wu|g4zQ2LfFPXM$eUrt+2y%4R)C?F-BL_BhYi3=vw#3@ zMt6LHt#2#jQ(Z&9|6o(uRc!-(t%}>gK1J}RXw0m4i6hv~UJ$5!0`0WU^JW7N$=%b< z1m`ficIG4Z8NSXAdcnG;Vuyb30N=Bj`RouHrLG;uC_%G3glADcXF#K#cywZEiYxn(npt1!;|(#+8C^nu`&_W~L5@c9G?-n@*}EYICx*#!jG_OeS-h zlk}t&jgFI@Jj}@C%B-oU&R#iB=?c zDU`#S5mEKL>5=0~zas|~pyGiY^dVGnuwz6`GBHf~hi2BSL6fF6Fmi0ooRr1PyyCbi z0x01ZX$-uhixwvpj__&Ob@AY_LDispC*Bqayj)+9G2R?0QU~oR) zM88kwWSJpkDM9Ph>3uC?WfMD z;9Q`Hkda$?5rj@HD#5bDsfPsQpV|*(5mk7y)*^B{W!{^%FGs0&pPa=>`Zum7-~ZsW zKr5!Q-}whCf;=5{sNlYDe0^xVIyES`5P_gUgix`g49HyoTgPp7l(Zn2VezC-w?%6l zAa?gya@~iZ2>D=y17i9oZMgYLE_2;hJpp;P^z2a}?M0S_NrwUgLhR#6zNta=6hTwC z!&7jPvDcVbHY~{kil?#?Pw`aLWvMM5H-*kl~lZF;& zL!a*PVNjfLdiNmYrhq`_(jB3OIrhyhb&tk$5HuSj=4#rJZg>R^wnLIR3M0cSnWK6{+uVrVVzb0 zPDv8vZR062K)YnoVYlhepe?&>Dwme>+`GmFFmVAlw&X!Zw7CvT*+@j%pU}8DXeZ^t zrxGIP*<5F|r}E*dn#mBZIZU_DE0=vb98;Hry%-SOf<)vri`NCP9Ekwcn>@J~B=!ov z44LmMxs2QH8dLc@{Y7{Xa{hwu>`O%PPoK+mV=jSU;<6PvN04?k3k`Mi%VWXev;9?4 zpax*S!B!lN%;SATcX{^FN0C2W}80q?lQZa2dSHoifl0rTnnq*tkqJ~R*-tml<{ z#gcIr)^hJft{AqIN=4TT!-yg}x@-wGvBR;r@jKoR%iF93)qPg_ehM0ir{8XM^~$W- z7tZB8p=KO2Qw>Lb;?^?>!q$F6G|1gdbZ*+dgW5NL&~Z3!oLNdF%?4rlF*$&x2OZR{^ zig$hEQoVM1aX8C!Vm&9a*V_DBk}HEYVf%;=^|@&|EOG_~dVRGn0XUQAXg^@NfM74y zd-87FLt<_bD~uQ^#O&N=m~3IwkvRBM>4S#5h|l%f8LsTxU03blvkgr%3~Weq)eoF9M_ zKv>|x`kjk?PnD(93QD>+ERa7L+uFU|27J>K-UM~&`) z!7Dd`qtPQ%nR?vRSA|?Y_uRcpwhxOG}^g`7U2$-^;@nx(|Z;%%g}Z^#9l~e~6{)jr&He-m{My85U(5aoJ?kwK|E@Z; zvo8YTQUWT^g~gSExbo}pMee5oHB{aQ!~NJn!R?TCtaaA%7VL0^{c~0F?w7Jfy^y5? z+E(2N>3ItCYtX~_m^iUrT1Gc=8>!(9wt?^CD;WTjtdc+jOyh+a|0PhDOWwu7M|{L1 zPEz{=ZZxXF%3))O>H7Y(#zEv(ZYBYqTAw7EdC-DZ1GuyPNx#ULjPaNb;LZd&VJ?UI!!L z{rBf1*Ugeek`+k{+%JhQ+ncT;gNY6|%UZ@)8A;0v1XVF)DRuSjtL1+sZb!={Bcrq* z6lNJVbwi|XF=E%1LS4v-+W5loZ86GC2mZXDcQtP}zc~SQq|omPAmY3bM|Z1TH3%Co zqth|#gj^0)_abBstfn^6p1Ejc&1jWWK5*4cD643DO1xAeff_SoA5+beZ>JsiS`cP* z?}+kww#E|>>BV-Fw9XPrM(*DZNW5l%HGaM*gsGD$?mZ$zB<)-ZKJaz=E35LDk7wGL zI3TIr5k}WBSdBp%w0HiE_uty3za>0tqCcShwA=7tNrgAgx24|au~NBYQCn{iTUpcn zfm$`51UY-;A7XZmv?YQUVTCH&GEz0w0m1VvCW~0EJKA*1YVNRXgy)@sHm|EG(}4sL zL|(W+fetWrS?;<98GPnocHNG_Mf#fz^KOG>u2{_}d=CsUm5}-uJ+|Qd^$giCNg~`=}dL2w7=b*FeSJ^1t+a)+p!dYt);(sGh)IdPr+QyQ$7do180Z7*0`8P<1ZpU z#jgaG_#l*|>;=7phf+Ho^$4<7%3sXh3d89+0+$NejZO>4YyNgp))zch2Y8f>&&ezN z4$&6hFYL)^DUH+cs9AEzriYE)BQAJGu}yn=@uk`2Un3kM60%v;D>BW*GQs6M3}Y79 zzMZ;^ba+Ayv+(L&lhZ)cRlT8gbAhHX@Fg`Z0Dg0yzP78iNlN+6BjuF0e9-3M)XlFCKdj#g3j-J}|9!J; zaG(_kGlXV&Q)P+_hm)P>b0940_tSQQM2?V0E`x~EdVQahojIF|pYtfT6(4Y2%oFKI zzE@E0WbF3&8Gw>hVlyO8T&x5~dl4xl+i#8x-RFYaQ*MKJlO(H&7N_H$qauIXX0>M$ z>N;4+W0rTL>-E+m^98bnvGt@5`_5FX8hu904X&*i$EfPNuxL4F3eT{aphqgzG5~k! zq1S7e4Zt5^p(r(%jzO&_@kz2Q-+r@{i*1d6O@6jOdz}=4XUTm}=c_=34P%$rO{gWg z&B(WIzM%v;1M`P89v1hxE<|fo+PhvVW%78FT*~#L5_BV;n|>&mr+bTR{Nq^a=95V- zrr{nH+|x^IjE3q27s(CIPjd!N&rj72k}~km(Y-5zMVoYu%VVy6ZZ~j050H!{V!oiM zu;we*Dg+4KFTIs7Tn}^$5TT%bKEBr%ym? zrFcf-M0FJf08ou9vLVYG;fKF1TsW)foHWlX=K`BZ!divqnF4gW+48~sh1lIu3nV--rd*^zH=NKB#jvkv>YQNpY^ z*yNS!q}ulmO!^qYzDD@<%8(c_lDh+K*yz+*8fEa>#nhSzr<}cfxdv6?Cj-JD(%`Nm zt(h(up)~$kqx)GVYWv%(DS)XdZoabtp zS31sCj+GK8Y()j9a}swdd8k`xL?GgR)^7ueDrSM@vpO_@=%428vborOLEto9RbS2_ z2!G7I7DyLxw1`Q38K-_nmXZavGLASh2(Nm#9{B3B@ma(xVx{522(8)U;<48h_0-)} zn zdb2LG?ucS!(|rL6dyDL+gCRf{zFDn9{;UdfbGCG?scq13K1~G2LEcP5CqGHKJrHSI z_bW4p8^=W6brHu({yw)B23Od85Zj@B5Ig8AVz`fJ1b<_(=~nPeN4jekVNxGdf2*@6 z1?gptwVCah^k(^YNnO7!!fSsR(bQwsf50a9a@q=3q}8)9flZa!D-SC-UOrv%WY1C2 z#|zxcaW@s&?@=-CJihbDAJ?DYGQ!!LGFN~lM!USwyDHMQuqDTBT70@tU60LQWE)Qz zhD9iiI8vN3jljK>qrHvViT_r-<7sfM5-$sH8{q~3m){Hi=nGRC2$9~g{+$bcT+`r zEtz$;o%1g&_yr-T?ZtR?wed$jC1ZX8*K-DVdgP}ahdnDlCC-bv^yt2}r_P9+#cDDO z&h}WAYhY~1;2`wW0JBYq*!Z7(1^hpIzTxYQzoZ&vsE%#9-`y^yut3B+?7hRuFa6iUZ(wPy&U@k%WBM_RxNIp%pI{OUj{K`S=# zTeef?svpj^%_DvR87P_8RN|i?ho=gw=i28va@jSksbr} z3pJ9`R!Nb%EvOoKb(;`;%$dL-{6g)1pa0c8=<_1=pM>F2w|98L{BV*fDdePoLHYC| z(r55cWRRRU0XR34yTj<#ItV3HHdzVmsVX$w^H0J32?lOZFus2nygCIS(Rp^1`spvn z(2}%h*#uK*xpAe`HEaW6jbzb^SQ>l{tGM_Q{z1sw2%7J%gc)zfCZ`eBtkNjMa|j&! z72Wi;;Y$uqN6P+bL=dm2Hva&dIe7)i>=X#In|&p>ojGN&<{;2N87XP+0q4Yru}J|B zdaNY6ddym@kgbeq#m~Wv3QhKSkdbblik@4-$hQ=%&gg$|S{L;ybGQB8L?luEf;yDv zh8oss9z0vbYA5U>9}I7qa|MB6yhMp(a~3rJN{Dg}&+WUbo0#_XqT5*FzTsBg%6Y`B z$s#Tz2RVr}PfJUr&s}TN2YB9N6WwD51|n%(E3(7s0V3MBNsgJX>#{Em%nb-PXtgJM z(+{R1jc2;6jL8>ttyQjXtB_Ev_#?)V<9OOkB(=e|F!ciY`II?|?B5&V;-zKZ)RWp3 z?hKMJmZ}e8;-00t+WdLFaA7ThNhFc{_$^&dop%=jb_6zinnbP`RN|!hH65iP%9?QU z>$ax&{lM*g>Za4?_Trs}u8iyV-w60oYyk<~lNsV0*MNc1NH*K5esLVzDzI~H{vSEk zJG3Cu!`m|aFNp#N`-7NXuOgCZk4!c!6;XDoJ@fnU)a^uZH=co^OBohz|3q|=s;)zX ziZ6e;Oc0@1}b60NE~<}jYA-DsX4`IF;0w^%B`8UPid z*!0FJB%QXMR5>FeTOI1jLWzQhIL+Zk*nw>@|eU^-hAs1hP zJ=fXS1+A$r1WRd!|5&+9qS+lBhhjBoI+!T9iw-ROG5SkOL4hU7+JvQFe2VZ)z8@lT zkjxNf;xF>;qlY;K6)E+-ucyGGH=dM)v)0{cz##+8z10{%-)E=J(ce^&q3^SxQ%uOe zd>bz`y(YR#m(O^4%XzuYipZ3d-%@JnY|FZg6j4Bk3R-nddR3U+mGQBwbN?TP{{PhW zu}XngC4T*d|1T3I9ORW%g3gUX7`Au0FA24YNCq2c9Mqg})PfFgCh>SrJ?Acu`CyLw#`omi1O3YR z=G}cW2h#foYs1JlrNd%1cYccS8`tU{l&(@43a4^Dz<-Ui0Sp2l!e8$FeSt&3M$u6> z_AL0%$*%Jee8lRbrWM1t5Wt>_6v@b-AiMVVYRbfcg;%b0>95$0q0VGpcULqLMx^r_ z@d*pR$WAIU6c^C5PW-*5_%E4IG9U1cbe<3Wboh6uSaiX-8XzSKC4b=?xEzN#e;2!L z-d7Ub{zBo<-U83I><35G=e?JB`BM!H{G?6K>!<%_FdE5V z^}YM)So0LzHP8nk8jNC@=~HL@Mhdy9k|g5mOts%XzVMq+S>0F1xUs=9T+orfgn#gK zZ|6^j*elys@cWy!j$*w3enRVjwkn%Zvx}46mosRI(oJgNsx1#&F;M>Eu=(%&*@L9& z+v7bNT|oUMwtWhG@3g_HnEh|za)eR;xKy_mw|U%Cx0Ouu z$tUbr)CNNT);76VrAf9U?REC=HR@T+zguY+4k>#Q?d{JdrbkVo_}hjP^F8|*V6(}P zoG~(-FHgVG`G;O?p}*Ce?P_;0P49EwYP)o8DYf~p;pT-6uA5xjZ9X3u(3{!kUz0V% zmCKw_ji2WGt0-{@Zm$Mdi#(D!X^o!`JUPbmt4ayMfBQnaUFh&9i3Yo1By2PnV^{|4#{Q8 zzl*%&D+C+=cBP5w*~Z=7$bPHRud3DdFr9a86h9I2Gglk46E4lu&KG4fp|w4OktDq& z0WV3>@qKaiasBZ-pZ{=vuQRh!;?mRY?Wc!_1aBxC zkXdbEcsHLe&G3A_-e}+&N|_PLp!jv>8W3gRtyq$JSv(LB+;Ri|<>lO82`{Mfq5VXpuQoDI)LQMEYQJS^sWQm;f zes28ySqO7i_mlFSfxsJG-9N()9bh_fZApsz3w@r77RcbUV(0!PkTs7}wWxo#JDEy*D<}aou&g#=8mcCSYU-O?#3MX>7n)8DOO4Cb=v* zX4_`lOO7&K0WaiqWTg@qSdRke%;e>43 zP`zMdT#`6Az}7EA#+Uf|?NXR?bhT}m~dqnMcC z{rmUxT$cLcl9I?jSyofaNfK#bBQXE@lJlkw0QF299IAZ@NXA=~{1HzZCu110pg|#R z)6Ks9ZoUrhDCM~lKZzW_aT~oh#*vsPdhAi}3ru+}>R=od;cbKIBEj3k&d$zStv;7t zgjESJ{L}8Q#{zb}y>T{~Wr<`~`3lv2&z7o*HuD`0U0R4qClExW@~J#8tckgF(xVHD*s^h*6Z+b9Zc)7s`7?;&CxTs`Xu#$VL+z;U7Ssg_3S} z8g2Z4y(Slgm3@(nr`d=oR2-s!hwnSXkgL)im#-*-A|{*fPkhhWy}KJ1ziwM4eNf20 z#_^#X{<8OSW_EI*@mb%+%QxRtDp9Dfo+3)`PVTo*OmOBlvJjfD&XhN z>ldZVA9<{;Nc?9WQ<}~NK1i;#j(@>h+$3)_tu@bfV#~3{&I;+m1H%)-OpXJ1~Q+7GC$7g4e!#nJA|g1vd-!uu0XUA|)WRdl6LV^c=M z@Y^+pS6t-i>_;6ue!45FpSphU;#tyk+40^4}HA%kJ1zOGZ`tD-vk50}VQXi>Sw2uIuX z!c47WrdTGHeyLtX&cGAHhN@QaT$1A_7z-TwHPDrwM@1S^6nxK_hP#u*?WXE9`Ryl} z@bK`oY8?#Y<8)@OJF2Oppa%6W`5V*C`aqFs&J%G&_2$=joC~$i=5H^Z3Rc$|E>=ZMs9 zNYLTdgnj?m`NeE~*-Bcy3vdauuZ`soBg12MS(eWUtLb{FzruK#cWkb)fb0=3AiUXUvQk1A0l3>HRrd3m)@q;}>Dy-xRPhCJ{X zCU`Gj9S=uQ7TUW0sKSeNYx!xSY6DZ@=C_Q#n`ril^h@vOWplXTS| zy1q1ph@844X^R^%N2?lgsIEVbp=bW_PMff7wcpvgP?;0t^p5en%O&wczF%T;Cw1Rh zmDxRVY5x|>1_}1VplO4vmhN;lJnMhIHB%%etcAsZ<&w{YXiBfynz9wk{E@%?KCj9! z?iuzQVZQl1hseD+6D`ii_lu|dy58{{CEUe~4TuDd!+f0sEwBii9O(b-=y66!r&qAD z&)RP~1%3_korDY2+$Me`MXrrLzQ$N0%i%&0HjXeqaE`G+u4E*GajplR z#2yQNDT>Rry4zpkD4^Kf>2m-4%2%Qa;jywe>@V;Yb4-?fOr4zz%kxL<)x#fsJ-=TU zGiV$@*^O$x)8t$YIO;F`!TK=COB&XqJBbAj&k18}rMPlQ)SD79Uwj+Vw{2%myOuxS z*X#DZKYb`UVQ$1`dtdRz{raav!UgZCEM>>~y4jQ%$piH7Fh$CtJml~l>i%5r*K-KixXhqf5d6S?Qil_CalY`^4%L;3k>+P+W>yAHFN=W1 z%uzM@NRQO-AV}8ExKx4B68YER)c!m&AF9PoOtzWJ*p2m1pK{>bm#d%K*Eoh{DU0@1V!bxd2Vd<4D*ZPxRu4US)`Rx_Cqd6qTEH z-0(QxtlgY6_m#q+pP%hzjIKKNQ!NjGhPq215Npzg`U^7o{d~)u!7`94Fw}Bu*WZ?i zTbiVkq!TZVUC-5IEKf0BBay6cpOqZ}i+K9>pP7^k1IM-;tW|>2xS3x&BQ=?Umi;GF zr0u)JJI5~$mLv2iG-! z1?jJ(KW5MqtGG&o;a?AO)AVmQA7_Z!LbS?&>&JCV9j$K{m^%Sp6oSd-{Akm@A?wob zd8>n=&7fqG@XHT}-CN&3CISJ6W9!=fbO|beEHdXjl@FFp5DS28+STdi#?71ifh*VA z>SE8{I%zdfJGEhY$WuFKpisg1OQ6GUIWmZ|V3bb9X#1@5MEPuieKa_P_GJf3b23p2#P1=65;UMP{4;Ch{?y+at2*1}p&6 ztgGZnW!{rYOBe8)7Bf~{+^wN;UaR?ui$fCERRK8j9i}1FmgkxJ2{+82vCDdDs0q$t zOL5ONy=&59{z(~dVa5*DZqa7AGEw)G?B{O<^6aw2rl0iD38sH9BRFyvd7|n`9~V69 zN88qFNZm%9wRe|U7%LbY(fB|vU!8p))?%7}GS(K(xltcnHVBkk7_-~CnX9gx5z zE(kx058=8R=l!A%AY)cYn4}uEQdoKU?>zCdu{0vrp^qOUvUY}&VM4LJr?husKn{DC z`BgDz;$d;wytJD8hHuORii#aVZdP_|o*2;#EmL|zzk?-d8NXv8(M_PZky%fcu;_)e zK^ks^(zydI%H}%gVqQc?9JDv(b}Tt^+gY*-1tdf4fXM9;#Y?2HyAS$HPE^UcA28}U zM`6sP9TGMPR|;Z$p!97dN8cS=PP<{GX{kGaYyK+~V#JJE0hZ8JL|C9@Pz`nXZR}7C zx=GAAOr8a>^VXI%zV=`eiw%cMi!wgPs_6JW*ZkvhKrmFY4j6uJz634hBS|1aJ@mEb zQ*eqWA6yHj(P8UFz?B2;pCLP}NYjd(1Ut}QUW1`Op^BG3+8(pLgS5f!@Wj*!u z*NN7vGh5}-Vc+hDebJz5Dyy&{6-HQU+Eg{1C55#L)Mpwf*!?J~Raq=QsUj>VL+=B! zn~J+qv~7V`jr%Hsp-F*SfwAU!tMu>iWNC_pt;1VlL(;Dr$C%uq3olhNT$U-f4M#j; z2yjMXF4pD{-1i*oJCa^gb&pAyqJm;V1Z2YIg;XgJ-4AET$pm>GMhKLf+&)O$r~FZB zXiI1w+-WzQ6HMiEGABe3g`>VbH?Sz(vye=+IfdUInjWW4bDy6)WBRBaW?++f6-N*( zV^v{uHeKsDL$zw$_NIP;rZnGYr|VliU&9dGxCbt2e(?z9Yh|lByVo=E@0tz*V_HO z-)NL9vDKBD_JS7?YQDD??-SyTO)0WqVpR8r529Eqw=voF<_Y5jk_dm=-%**uVzyrytkY1@ORF+hri~A+- z&<5}x^R7ku%heX3UJodNYUtE`?1;K!<7I-?_KU@@nYeSp^dm z|Ks?xzRu&(#b!r=zO9nk)oAHCgCDXSRgP3#WJwAgw+V}h);?zq#Wzykw*~$p+f{2we3-48YRkyq}t%G-2&<2% zlS}njS&39P$$W$&mh(-8%O^I6O(#MWHVQT_oe@Au&TZ)=i5(c4@904SwN>!#wONiA z$|G=UzmZ%R5pi=JFM3nZ)W2m|y}OsMO5Jm6$P?1Jn{1U~BPUpze@pA3l!gp}XOAi^ z5i{NAdRl*iScP!CSa>R<(hbdQF#JL$m|Q!|-f({DJb$-=#k!O0Ybk6mAk6smTQUk) z*?0NKZH+?EDT^c=&HyZVEsvRU$eR{hZE-yo%-V zqW8e~)clxBHXpQZ`})XclS(tdC$99?uWV{qP4{bjb^@{-5|&WRjlvS14JwfW<5P}} zGkRfT98L>~!FOk9M3KGd{b`*TN5}4U-1E5p*%a(&-QE6}h*KGIpDL}>`z|g5 zmD}lwt=@zv1fjNqc_=-UiHivP$hp@cNv*I znV6SrV_)KS21jpe5yv64(pTsb==tcQcWwsXT<#DYk`a~`U3o-TLC5+VGZp@HD&$g& z)@Tm?bpqXi!R{@?T2Px5OI8^;)R(0dhRL){oM?&gW4zpr7B0ivAUrpmY5!a*OVE9X z)6nVMNn}jM>cQZ5#|mWZd~~?>{S({|S1^dYlb=c4cR}vv=@z%FsFVhVFVsaV30@2c z9Nk~<9y+(N8FfHuzTQURScvWSw6}g0=zM^ZGzvayOKMOkhV0_n3|_m_h|EFa?!?9N zX#TlvqeWNUPRl^#=^tU?@P~^@hzMhgPPf;!>43-!PSTJny6?Wj2>tY+lePF$lD=Kk z#a6^H8#}kS9r-VFBjoHC@0b{PNlZ;?^ihDH@fhx46_!Ip=4q#Es!+bqfSE!R%IbAbD>yY01% z!k3Ru5=Q|8JdV>Hjc)^HJ($U3%|dF@%iE^Rv_bo9sIN)S^{jNz4co!|pK~4xM%^BO zGHAJte?W9m!5DklY6FOex2Ja(!d zeSTn5_`=puPTnw?cX5;}Fs;@)UUXmW^Tt%Ox|#@sW~un6Y~zN~OC4->2(5SPlMG)j zIetrvI{_S;4&sfm47niQ&iy47Nk|=hZ0hZJSEyxkseXt{o!#Nd#7cb_VN5Uoi%Wy& zqJZjGMaq;wT`^N<;(#u7j5_=&oC4mI=anUCAw@h5Zh!BYg4lyJf_I!zkUZ{Akvw8- zmb_a=LyI#7YQ4Hz+Ajnaxr6Rlum#1owA4E zw8BNG`LeLpc=orb1cD$}t?E!ee-=_**9J{vTyAV?Klm6A{u)z^N3pQKFJ( zQp7SDLi0pk^oxH4^7w+Na8MLliMH+hNynd?xo63w@ihr!_G8katnRrY5$&*%6rG2l z+w+xVg3O73{QjE)Z>WUUy2-=5F4Jka2DwDB&(6rh&h}}#!gk{M_?B7I0YP{d8e{62 zBO#^JFD7J&J!6q|mn^@3!;a*aYaRAYfu7c{w=KkLs5W*id;6Uix}VSo)n^%YBA4Gd z*LkK=XwCy!VHGyBxuiiyC2czlmFU?PI=-`Pc@ZlA^|q`y4txF3PYWLcM@z8)KI0ce zOegJIETN*I))9f@&0q>RYvij4j=l(+s3qZ-c1_n44zGjm?A1i1U-7Ydr0LCI*p;^7 z$B46bZ>Wc?FrqVc>Xm4>54|NZ-VK@(eSvLRgv-XyVqJCjB!iNZpArtD) zZ#Uw)NALrr3V4!bj8Cnla9-=H@U&hHMdit2|A|P zJ(0>5l^$FXFN&=PA21xwX5C)zE5V`sR5$bl9|!p+K0)ReQ*S6{WWM5WU+v2vZEzC? zQpGA6K^KL5MgnRc9MhYtST-^sR4fD*>1oENAv`ka?jkbSa|1V>acp2hVYNaQIwq`y zP_-Z&D0L7d2vXZIc^xlLS)nkfPH)Khto81@VQZpXV$ueuuHE%VnbDk^YVc2_|n{w@4xS$|G zd<=c60>2Yf0~-6`Vr`JUTg1r^gFq{CsDgsc-kj%K5vLB%jMKUBo8HdY1}vtSBb)*e zZ)xc6LhVOs0#Qd3&uZ@ z7JV&;p&%1eHiY;70T7I1)-XqSR-G*(JmwAu!-79I$_viTMyfuX=X}_HBgX+1?C-F7WaKB zWPUl{2C+ZZ{0|bNLUlm?Y>bpNHEamQQLqZaVbF#P!LgcN4NOTZ-B$HK=@QP4na8KJ z7!=AHuYQ;SyENfyW6RvXQ*{xD!9Uc{eFP_+5NHne2uQHvT5=3B(()1V+&5aklGiY6 zHRFIsws~QbcJluaSRc-JVid+V^*`-0r~P${edA8(;KbcnQK&-b8^(uGRVbnyh=jkV z=sKmT|K+0TD;H0VY`8l<~dKv_ZoDM3I~l%+!&B%}n9kZzFfhTpaK{XXC4eSiO6AJ{9- zIcMgaIgVrWZl5y$GQ8Ls)6N%v_nI5CMc#oBOh5yLjm&dP4E_9=to|2MO-5sAI+M4R zSisTZYBSSvx8}&x(V*T67_<8>gIC|=%m0+1cG>}wAc!RM8?xaew_(oMyXDAyc0)X3 zY~Tlpu{+H^l^B#KW{0@AN_Agp-P$xIiaGGvZTG6bTk{Hfl@6vMb9$`eCX#IOg`sQB zpT8^&>I#j8;6Oh?Szzz?pTzQYQ8JIv4m}3h#8c8hJ)upwLs+eqZ=S1yOHcz$cSx)w zL0I%8#<)^c>t?uZL@$VX@iK9pNbAY-SBdQ?azCSt{mmd6ksq)H*|3h` z&k1RIta?ONafaLsXq8fwh0tI!>$V_LJLYYujc_-3TQ*J)xBSJ=Xl=zpW$C4DvdP}+ zLiq@=YG>bG0fU9f z>XYcc)ekLlc>!-jq#U=OYr%x3{IBF?M5QjXQGL3-VH3t$*acDgcqf#Ag`QBh%DN@- znrd9II?1Z*e&jmT>ili%ewV(YdLBX&SFMeb8FXhOefDPP{mO0goqCeqcnR)Kl4uSm zAxS!#5lB1X2xUQRN;(?LQ|ms~Yy~*We|%A!JfEbkKOzSuV?lm$d!pgO!!Db>Z{Qar zuLy<+W1YY9yAPp8)x1LZjS~Py{Q?>YJA?%=8*g{5?!?@WSuW6N{0!4EXCQA3P$%^U z#34*Z5(c0M72F2VqUjaTUKae+ew6}H7r+Xz35O4yGIm?Tf5 zHUqf1+1q{xp~T)bogXYnr@yeL4JE?_`lpl zvlci)Z|HfKQNw4-uRE-ol*tAKO+4aE_;ON4Rcwg>$Qf%G4b;f88aXLO~ zirGmd-h{9>BoQ5V);Wmw8^KpfsTj;4M=*jO@2Ko~EGauG>K%-wXQ5k}lu9dh8u5qk zT(0;O;H&*c;3-)Vaq@x1;0ha}i#Z8KUZG2y#VceJJ43~jsN}hR>=UG3#~hPE|Fl;~NqYmkNWyW{9>YCiuRP5d zM9u^kLV9Chf>JUJS1I#a5J*1EcsQbEXGM(&3$>LKyH1c~dk@I;LL`B!zbM5{6JRa1^v z=uG-K;WD-2ej^Wd_c(sA2=uMg-hJ``YP&!JVvhI72q`ctJp5-U%r)J#s|DxEt{o!vRZuH*4Zsuh{HW5}m@I0T4#`*OUeMgD{uLA1`pq&`!|r7ox4FikZ1I~r za_Z!Gd{kD|DHI8qeEjZ!X5|{GitXam+jX2l!WHi~v8`0Gb<7C5|2*4PFaxc)#gXfZrNy-XK}wZ_S(D57)) zukj~|=5q=x5 z^k$Q6jJb?d;ZuAh5(W-LJXcI(*HH9|qz{R_gBWFtK^Yp&TQsM^-l1OMd*TRMxWonR z+jafQ3*h=&1Z~H;hNjEr_RdC*i-oB4SmkvrF8ee#-lPLiR?Lv2ufy0kv0>XT9gV>tG$UzBi18FCL1XjU*;P?SZdR!>DKek!w%(2IID(f)V$L z-w<=xUIvg-fD>V$eP8l7Zaf)i#bvxmoJ9(Y7H}wt8sRNzkQ4}`gOoM4N7m|2+zo=* zu`b2(eNX*E#GL(54{VL?a=;TnM8cG&&4GuzPa07cMOlF38#9mJ)^DUv zjxW5qH>hm{D*bh-5Y4YI0KP{2x+Bgq6Jhrp*i==Ph4*o7cb?`%^ORDS`AIt`f*@+< zAm^^wjIKJUAH)r?F)TKQ!)RuqU4)N6iEn#ul%YbLHAY$uwmmQt2l~NO!CKi2Ucw>q zjzqRj=labCJfZu(1HlU*x7u}>`6jlxql!XMg)jRh>w`XlZ@218o$yjx@`0eEhG+lf!sCmNDU96n3pXf;Zg8U445;?9M+(uWx&w zZq{OlUS>gP>_7Y%;;l6;>sA$T+o^>)hG|OTxMwo0=(aRHpx}A^+B<0U&0*5zn%)2f zSpu0QnG^CsU{PnqW7Anuw+f4I< z1UTmFs4+Jx!z%YQNRJZzfPmOjDDAoY;WOBVs-4azz5yB!)q?s#>tLPgPZ#9U(t}`1 zFZ9B-?vB9F=7udng-J{-B#8y)GBw4kE{JBhfYg2j2X%dsestZG%Q2t(Y`RYwSyFmsyx_SFlHy8L?<*Va{ke0fJ%gY@&&dH?Ki{&hR~zg#u>)4`5$pjZ;;j*DvLh^ULf;nI z!lAe25cjN7T4Y#(RY_J;QV6>qxDKefZHHLMJl*y{HfY8yyQbEZKb{RjB9!MBnux5q zCT9O$Fnm-+k0WE$^8aI`%ormj8x;QiBSoRM_wkS*<&w}nhfG|$>p-im0)?6%8I#-< z6CloR-_pBDmX=Np0XIt z7w|z=ieH`Ps#(mkR^roR3j#Xf@bhj#KWB?f^lN zAEerdVkW{J#+8bt{d7eew=R6f>$sZ*@CsB(1Ac_yiAM#UsEbFzeRx~?J{E;^| z^ukHgE16)aZs}DeLg6~*DzZPCY02tQ8CtOg8sn58ZwBZ+ja8080>IiJKCoNvU-rxm`7ouuL@9+hk-(c-pG6>qqJYAdLr&Yz8~hwy zNf9}~CEhlTw%^Bc;2+VgwP+(7@?E97gd4RU0*B~F`u+GBzwU-8sd+DT+`M43{vy2KC$9b;f; ze_&4ODXdF^IU;cbH-g5P~k^gAoaY5 z!JzuYnDSq@-K=5L#UeI&)^k{dTWP1%h{roCQk{1O`}S$RTJu*D&?y8-ITbU#nssKs zt_8rK4~#YqF(6pz0br2efJED`G2i=Yeyj}Lz*d$tOhpb2@>u9Dn;>c!*ob@R6pyN3 z2@iW*ZDM`>2=;ezoU)J#9Sqilf#Xejgxsc@g7>{oH2*ft0pH8?Epod47lMaRob6V0 zm!mJfDyPC>cdjc{#9upBuzam|R0RIFcZNdyLc)8``Pi#XknGSH#I82`VpY*}Bx8Pf zpdP<%1ZYiR!>QYeQrXo8#hEiv7yod^3rLpVE9jApvXLlXu zJQAPBQYUQd%g;+BhyTv1AyUGvpl^#5ULw+A;OMFF&`^rO|J!v*mt@5PBVisUCwkBl zz~Yr6zvQ!tgd+nRL-wf%ELZPPFkKz;R1tPnVMEA-?@a0jAzmsrM+ryYv$(d>3S3P34Cyrj^jC1<@8X-Vins|^(w+q)dQP$j4)bY z`E$_Ad)UNq+(I62>=)#@Rk+5x$j~5blqI+fRdbZ826gFB!POTu@+bVQzZl{UH)d`y^*qxtksMst3DAjZ|0>W-o~Iw`xuz9Tv_dTw1s;o zX*-s85`TqX0|>_@Qm(Uz!-mx01LZB6WGkzBsCkIa=yZ#Dg2*{tzFzLv%Lpm^ZCk-D z>&912!Zd#ndP^ZOE#;O79f#8`l2Ve3zR<3>@Z}XBwB0Tad)H7VBqE4q*Kg}#;%hOj zz`O1(1x{f+dflZs+lG!=y!&qZR$(7WIR=uLqj?>zXY!zBMT(O=oGf z_y=u^T@PfN!fy2Omuty8+ulSE=6|0%uMjP5PgS|$(~nf<9*LcIRRi^6#WoBadwzoY zXwV!$U4K}adU|#Fv0@rY!YFdmV?jA!bp~Oice<$xGZf>VV|c3;lbiEdb65G9+V(XS zwOJS}6-o~yfhoDyEfDYu)s4@*2Tu6O&}QgsGY#6F2Ws|6K{n9YK8=C)JU*!uzB)l0 zQT?5I-1BHczK;)~jvd^nD}<^dmaAb%74@agPxfpPpbLBZ69=1PSy14Ubkb zTVPI*5Dy9OWLt@4FRvR9n3V@u*e3yG7twg$%M9boPfk>g!u{Kw>38iLztgjFFmQ-+ zFf6Tx$>wgfY*Jc!?Xkqw#Twyp$@;+-8f{6&+=XIC5hd1JsP%psSEW`ZY31pK*Spd9 zYm>6iM(=HYzavGgt`-If$q2fM)ry|rxF%xx)lah2?oD5jcH+-0f(pQq+4)Y%K!Nhq z^8-vA-Kt|5xWUOwF`rCcS26Pd&ZZYEJ^cVFbNMm&Ru`2SY z9+i@KbnQU!(IGBGSzM8=Uz~+(EOJ!kYc`I1KCqm4Td0D%fJ*D?E9uOj0#-0#Hb^~Z zg3+N6tpJ3@LTgcw{nb#g@7Ba{ad?MslXtw@_mBzRajAv=Nr_9mO=buOkkxq$I!E~o zo|TK8Ku;vQVoTGIg*BeX-<+#_2#r=Jyzhc~Ws!1IeOEb(wScZeX#FQu1Y9Lt z@GtoFS4BRQrtBYl8$28Q5n?#USi%CF!}gLTpaWqIJ8_>ZALod;QPrDpu;8=&TJ_w~ zO#OoC?LESp626swdXkvOP%;=8mcM7fDHw`gjK-HWgQY`v_IKi_BJ3VK3eWDGQlL}9 zox~N^31mh-`fBa=kz^6%zI{ctgWtMbzQMl)^vXzpZ z67*07i|r1a-0Q_9y{%OQI&z&s%Bzrb9w<>Jv*YW`A4l9O?v5f(mGoKplKMDs#*w5Zr0_TK zOj_9n<_(>uBT`vlc9Kogc~d9v@sb$%e$TXp=XdIp;bY-iRRs6~`O<1`G{H~4;AID` zY=+>Vjz>t0OH5~x0;&e}=BtqW7S%V76ue8HCEHm#RvgB|4SsOt5iZb1OB3&L-@RC9 z65n(G`+;95<_tgRqBA?0C3Y>7J^F4m&gv-j*u7{$2zjabo*i>}7gAhlnaIp=$6}1b znR0}o%ny&AKObU6-Y&yxYTum6HI_IvWW*J*NsS%uvn@nM^>mUogOJT)q83oJ%s5W) z_`wu7Ub@LF-qAcL9|Qd^ZA)6QGx7Na4~f`ru&-jBPmqWY8x?vECSJ)J$FtR&7ApT(mKt^ySY>%{&E9Dk| zZm2)6cKf%M8-=Omwn^EWOxEtYpWAQH(91u|d{k!kL_pLDHMp@e)VQN*xQ!w@)T&!w z92oiHL(N#Omtm*Uuhl?SXvC)Y17+Bs|3NoPl*UA&C5}Cvxn#v|GuLWwbc*y|nC-ra{&Ik9Dwl{eMXu&+ zpWdLR8urF|B_VpPI9mt5ifeNV&P-!s3KY$!9$`GpI}gabDDxn8Be3e)%yVdlrti89iH)#wHjOq@mVn=pM8ohth7{_@#oh21A zi|hluoFon58cU~&K(Z@dpbWJ-+e*w!NJbg_|5A5b5~>I+`7^t?cGNlt^D(4%}H9so;pEQeM;XBnH+gx65Lmq6Wo9^BeW84!W&_Wj4t~PE;e%Eil$w3H#V?pbsAuLSA{_yeoYy-KgpLiReJn0{8Jm`&C;GmTlq#K2>XKLbEM zJfJWak9T6m_}Cg|=zpa0KM$b-oda%+a_>1J_~%MIGo}#ojhlOxxm`2x<}_>Nm3o*l z#mxz}c44>DjaEy=GuL7*ajCy^SHVq8qm31^!Jgh94VUJkR^v~#9CtOOaW=BRsb2H`d{wd6{b;hlh;x5%4%w`=>)xWdABUhr0d&lE=kspWag_4 z&YUW@^hSW@)p)ZZt=TUxR9JuilezpS|7W)Gib{|P0BKcx5IC7#`EwFvJAUGRFQ@wFCThpdH}>~;mqkc@K)%Tj^)75&Y2X>wa8Pi#3}YnH$hJmHw7o zFw*MT*Xl{D&_lY2v?j2_sb7E}^Yu`m8U%(8A1W24UF1z->38K1rqg6ae?n+QFh(iz&Lrob_x_(pfMH%3Pw1lRpNsNvz)V=R zpLnkSB~b1fI(rtC_%_ppTsqX=pV#SkYu-i_Sbgt=MUGeHPlj5I>5$vj)Oy0A72}_{ zb+*=pCB&4L#OD594UtGY$oA!NB5GYHvWCVRv%KC9Kj9i+T-rY*&+^oVp~O_b z(cd-l<%_~l!rls=_Eq!0W~cxxK^yDgBbUJ5i1KVrp^c9FPI{G%T8-sj z+5w73h?GgJRmS!vSm{;7$S(Pec8hqw@OvcVv*n2tWnqvDB>bv{S z-r$SZU7HxA*hm zcyC(LxP~&|rvsg3KAQCA+1an|gdCitGcJccG}NqYpk9ubCtZ&I`ukdv|IZo)vQ?0` zwl5k$D|c{a6?I1dEv>eJLa*;4oM0m63N}4|jM4iWJ=6X_MjwkY`uz80{~A5!Irq9{ zsk1EZewpGd%W#bOcsEAxy~)bczuU}(Hs>SXi^I-0)W&jKw%cmSn`$)PW_Ys+DAYG+ zkA8U&`st@Lbp)I8eD}&!DSB^Tcf+gkMODE1;;MoCV4ib%kK$-7bp#f03lJ9K_VNES zLg4$rG3ySqsy6w@EDMkrqrj7)d23QF{)>f92I!`Y)cFQvhK^$!6l-x?lDW<`w><3{ zXIyUX{ajcMQ5V=Sx4dAJcPvytfjT9DozQuP zx4HMGxPbw zDR`;e#p9+Hk5pMJGWSCdo-ZDM#!E5>Y zM?hECl6|ghLV&Qggm1|5m-MG?T3=gM{#ib9U31sZ+YV%IEVCb!{`1kXrkI@vjf^To z(~so%Fz8fI#Ghde>WGx+Rsc!KAtfah)C17=(#F2Lzm@;c4nF}VjQrOzfZt^FtkG1w195&YmS!EKd`v!k8HY@!_RJc`P&)fjZ*}Yn(Iw`9kx>+ zCY5lHXV0QRLGQJBaTeHx+=uVd+p^JMh^)WJ%DBf*-KlN{0*io9ho|x%)q-o2;NZrp z|DCLi`TnovC;|3oehF&a7sw8PBvcl&9I||x2yKP06L*tzHB+9hg4fukAHC*^t=>DX zQ@9j1XAxbRtnn_(wi)T2_AgGUY|i~jF?udF=a!PbTC8?4Cg;_aEVlBD`n$%@txY}l zf76{OoBzi#abY4vr}FJXjJIk{!t8@?W8QXoLE~m^7O{^}PreMK0lEOGKxfV(x@q#Z zxaEvnIoDrCU7X(Qq%g`Ez%9PjvA@;-%}@~0`fbi7P1xJOE@)MLBkU1B{cLUM{kOlx zsUk8S0A`+%Yr&lT-;{ug9rKD0QuEFK4G-bKz4a95HU7FLq`RgP4p?_POKiVkA$**A zNho$GMrC=O0@Hmf%0A{g%g^=vBg`+SjBx%U5U#1WhRL!qlsdVf)80(g@f2&W&uG2K zV#_!N7;q0H7;W<9zxTa|gN<2xEac7Se{7N*FwwTS)F)}?7DoWWKkNt3WEH1opzpB) z7sJ_c_FWGvW9k1wy0onnG$b!^F$8EO2-`yrvk`3B@>> z`NF@b|Fy10z+nYmo=vlz_Rk}nV`5@(TM@0`mFwqe(SRbCnR?YgKg~4A+L?^c&-S_6 zo=VY_rvZO;q*hy>9qg&x=xi6#n0SFRE;gij`O*@2IdL1t-$ZbY@=HaHC-dMyFh2;w zHVuC!5q$M+Yw|7kZ*pgW6=y?kMbbb`PbI6}*`^Aw6^(OX{!5Wb^|O&$cgq(E3A_R` z3H}PuK>$B48YFaup#U;VKSzl?{efsScgGE0$O48$y395j2Oh1%wBO!O9x2ic5*%&` ziN)GAWDUD2zq&r1^Ew(<)r=w_4Hl2go|yq;46Hs()w-w0Q}b%9{CNHHFmXS3G6|j| zs6Do=Vp#&H3C=IfA&eZRXnOZAz(0Q^0U=_%R!(EhbJecF$nj!XI~^VLZZ z8Usuk;pLVj?PZblN8i>kU7E`$?ZAgJUUjXC z_WS`>!8R~m=e7R(2gV175riygJfmjbgAGzt>X6)NcJS@>sTY7_{sBEm;N8Io=r3s&{V5Z|mmpeQks z6ks4~CeZK~0@UX7yWc)~e~+iG83ZUkrd-iM0C{{R9G46QP;g8{keTG(yY>V+XN7FC=YR(K<5K&Fm|?Ws&oxoC8O}sSf*P*29r;TOqh&* zM|Q(F0cgWDGI{mDtjko#rgRmq?KaJ7Tr#Qgp#!kXb3l_OhY8R}9cd1^WWwRu)o}8`8QzBng%dx*J^5B-r|FtTF}K%(@Vy+#3P5f z$ttJR>Z!s)4qhH;#m`lBt{e``q#ix*s8$&9Utqdl&8gL9m9qJCf(JYZnC4nz>+u>| zXC(m)?X4{!@&7drD2McPy<{x1XbOBHi_LnXxJzfw9CKX)=y$Mx0&*7MgMvdau2Fuq z*=--7h1+#TQybK1ai@+q)tJq8&k=Ur)CFfXO7cFY5tw3|G2!a*Hmp84zH(gCB*RIS zS4&N|FH+n7yRr;VCHOd(ET%a)?^77B7>!9X;jOXy+jFbXi=&v&cE8Vto^Fmjn*8Qn z+cU>j^yAH&cv5GzrskZ&CuL+j0?`pqJ->C5x8vmgz;x^ii;I8IJ7mMmzxfU*N$-#s zm-F!fvlxFCFTq|D;|a~Ks62-Hz+zw_FuVQ6S1S}G?d^d0g1~O`REnUkD|j04ru({p@wMs!NLWi&$VJ?h9(tKQ*qrv}PU= z_krFJmdZd7PgOOLxb;3H~R)QA>d8v=Xme)Qj>^#PI4oDEYP~j!B zgD;O>gJ5`?Y`D;i7e)ugO3GMDo$rP~&qq~)3N36{iSP35NuL5b1A`B^F|=86=?F1o z-2-zF2NLf?j_+G)+I12*QNedp_ok=cnS>O2zY+vUAWYrA3q*hv zKqcizF#-3TU|@f>I$r>T_bQ`#IPT_7syvo-$BIl(lQ_cny!R$yE(ZT31{j%XY@Hu^ zeKFD!7M4|vl!~&7TMym;PP>8;TWE_S)C7XU7ss@z?20i!5P*c256tY*%bC86qzL!; zpo6>tq*r?W@UBTg)#k5nfl+WqNnsI@>E7!l^rQ5qa1Rq8v;sBU?Xo7PKXbezx{UzR50ONouzG#)#95o%>v#(#jxLD@s@mU+9c=E*KGH;p{$xdvbY=h} zlKP4tc!j(ieUGUw{ZG0D{h!j4Z9$Q4m|8kx7hgI?9;-FiCpgS%tL$@4--Oi1gZ-op zFTKMbA`4wL!os@J$O%=e{;ja+7@A|=9#pQ+w)Nj~g=ft97$OU>clnyzp3T+v6=t4! zzAO#g5S=_YWJtYnbVv8yD0t;J7T!f|Jym91WR08fDTvxhcJ+x_ zzA)YI&#x^^fEE!bFbVy*H6x+N6Gm7(Q~P9vFQ}&`h`h_n~8 ztBP^d5Rlc-R1{|EQpxdm{zcxkioCnC5@=xlQ@+fT*|tFNs0ldoP1bkRzuDiHK0Z+3 z7NwK;=muwQ)Vj2Shn+<#U4l)n6n$6L67et8FCgi2*$w-jiY(M5Jnmi-+7dmf%Pkis z=z?4KCsOx+Om~Lt z0O}y#K?(>yeVoFI{JGio45+u$FPY})y{EzbQvP1#9D?3zGl{<@B{~Jp_)V&MPGX!J z7%ASSNetFBg%F)c%>K2~4Ra*a+TfiKN-EX&@Ui$i8gVUj_o91qRt%q@lsosm72{Fz zzW+FtbJQk$K=@R8>K$_}O(yT^{G>x(_4VEoJ;~AScJ167jIaXBz0lc!wUyS0^E|nr z8qSsS&zMi7ZSYuH;DNq&JC}bb%mlQ zza@d1!q*ojB}+pQS7BvgrcKIwSGl$Rm6&39qnD-(_sS;3X-yLcXRG18b`=hUWSil8 zpNc*I$ri&@eY9qK<6pW27VI7g`La43Za_?h^3;745RfS3S*Ai5mc9weG_F|rU}x;4 z5@Q!s40p4wP9`C}fzX2Ke*C`LZA+`48FB_ofR(^63IoH$ScF>uY;Az01EN-(!B!AsxA1tfR))MN&Kp8C)2sBE=?%==Z0Sh+};y zXlVhxrt3~tWpV2DaJM|m5=Nawvvw&)&=v18wZ=<`P6&m{`kza8Fu_225=&3FQ`fTP z*B;lA8yr%m#>c6S2^}VwD-JdR0vx_Kxa97u@-C2mGLblP5I8 z`Gm*geMpZMah*+2B59I`d(eD*PL0#At{PqD9hR}znlZS!sLfLHEk?#=EQ{ucUP*_8 zvCDDKSEMZ73r>i$Oa;6KWPN6Uu|K5_e|$q!?zdKEHAlWQ{V;LA^At#u9lP(u`UjQW z->?U}89gj@9wP`@l6FYi5}g%Oxeiu1*-<&7XDUfO#v-dC8nlX)OC5_1y1K`Q4%5k# zpZPij$fr(KujX(GwY#wd? z_KBkle+q7oqC%i)Xso>UdMc_hzTJmAp}DpL+PSim#rJH#plK#ajMlcq5~-P@LTb); zzU|rpo$n`#jDYmyOwdQs6rlN}R`A(}7Z7&}mrD#0=m;L1tobcTv(+n-7qQ5hudyOK zhuipmr73-gjGo?f%b_r&c=pF?2#}>m-v;Zf$)7Co+vlwvK3*Nv;$}2_^6Al6n=(Hg zoY{-|+*5{~$#UBdkA(8{c^-?CN!o{zKBuwb%aU+?BiEJ8fld=J*Laq3>MbvNQ4bue z)+Q6Y<_7+xozJjiJ1_g^!Uav^}nZ^Y@WZDW;CtkB6$nff#VTRHx&VEr;wP27r=fA14)460Nt& z(tM;S>&=?9BR`(ks_L@Te7RHENx3YEUmt=Ia2UKEJ0ToNbKao5 z&JS0vYsbN3wPTX}MY#I1JzcB|=Uk_$kVOh0ZF|oX5ctn55)Ft=lTT{hX7Ykmy0B(Z z)d)&GFf@Z!O=6WO`{HQBwyc&m`s_O$>=d?KLM{`|VQ+Efm3R+JzG|DA2Ki_{u(1$g zhbH-hZ-~bpdL|;{G9je-NTw0C2o?gzx8g@;j!$3xy1Oi2K<8tCW~40-&-hNOc+nAZ z(pS}%Sgzd3F1xHUpF%@$kRu-{QPZBS{FVI-^&n;ufiF~SNoKlA&S)p6)CFJn7K|Cs zrj@j{3AS1vqc2mw5sB;P=ip(Bo07jOY7kI7*#2j zwRu%*7i91Rk28dC`YjIb7!|;}jP_q`HQRk_!$c-?(ew5og?9cJ(B_*Ai3_k72w?tJ zkRBWqHB}w8c3%vMxP(wbIB`g>8#juxwq)o0mtOj6SAdRgLVL zY?1&;7ayzNlecl~e0G*h%j73iE|E>h^6bFBA8T~oc0pd6Cl4nW;q)OYK$-sN*OzO1 zOK`rzf2vCJ5AmRX!!FKg59!?^uAJme_brTTb<8(@Hs@qdUveDEgIR9te7B{IU6>4L zLTZhk0A1#-B<=nnA3(meu)Dn5E^HO+?(AlI=A(c)zO%8a>&jBoZx)NZ-!&PR61_Sh zaeWcYterWP`h-5az@1{Gi=iBbU z8`r7Y$sz37j|(A9L%mH!Yee0*Yht@T->=<)GVtXc>i0L>!gi~1bFObpxk^zDtF03Y zA)9?4<%c$#WuJb~y0H4ve`Ozq5rJ896SA;g(V%4?t>%qy4TDU)KCYFk3&?&^BlQ#U z=Z+ZLoG>AQF5&*%w_|!t*>VRY8OdkT4!8dVb3Kir4mYt{*ZBTAx#IB;&CK`}7dvfQ|c2`t0AWAak%y7Uo?1KC?q^AH0%4Y)87GQ5Sm&Ro=H*OQ2xs z2gmt8STfeMgcf=80{@E__bA4Sb4? zY+w)D2(a;|mO?MSr?dQB6$3avZr`@WH?N1zU|0}a{~T##VK_%&*tF^^&mCMD5&?2CK-NYG5D4i@5_)(RivH@R;^f5TrR|H3}$E*kA>Dpon#2M zXks?my1X^0BD3`iT8Wb$cDuXFFnt=-5R*`!a}{ZQ5|`xRF7Q4D<<298KP{2EgIwWE z_@aMp$vU|EAC`#HoKdul`0u6$5|p`uOc8hp+8$lCr!G;=>|{#zB60r5OD~fA#Bk_Y z#u3o@@7g!Pd!4Ian@&tLnZ60`UV3*$QqnlGx~OeNt4b7dA?}|HLfocclDUtUMrPxSZe;kK+#_$*{@V`fzm(XzCeZ$89Q8k3pTn(MzqOc++z88a%MS zcI1lp*isUGIvQ*Te&)5}>;QTawK?Rcv#qmdfDFQfPj~2xmxRUGw&pfj6Za2;0O>cp zCpp()O40bUwi7L9*19keoP3H)TPC1(GWd1wl5kuZF9Nrw|B`l?!Y!|TH5;|lREEBN zVx5nM{Jx;VzGZ7|KH6Bb>|uQPVe)jEhQI&v<9TpHd+-~@dy;^ZmZ61b{iBO8t~@>V zr|v;(GpBZcOnHMMt(%<#&YAoyK`tsqN9`$=%wmUrATHp2pAQ4fpFy z^3S!p){0m93^PudDQA2KCsz;E3^zO5*OI8%Ib=`U9aG5;f}AEt3M)kOic-foOg#&H zKELrAMzxL9XgixK1a0xAe&}&_*Q(C@vNUknQv0aHNJ(dNN;NQVR5J!&P$>Uq6_?XJ z_$@Iv75w|{`HbD-(U~`}V+4zNU z5sWiA)7|8cJiw*L-_SV1I^JL6pYBL9GJV)uHYQQV=k+kKtWvDiPT;&~+P_JBeMMZm9ZaD8L3a5(~rPu?{O*!Fc0m4<0 zJV-t~Itgm=OB{B~#MvY3|opd0=`&CNkwLM_xFMla*D zUOT+x+~ZV<03%XW69y!_iT?1k9%XX5w2s+53Wu+fjWiIvqmvC(9%8PLhQ%d@dvFUzSX08N-QGrT9IXPOoPKJiy> z*}{WcbGn0)+j*z`qoel&PU_dR@!Bcsw?;IVN98{(pUFay6X}*A{AJB24Na@ZJLFBy1wZ-XKF2nOpGPZTcD+uUVPuR^_HWQjZtqk1%u0{cX_u5$O|1gu!ZG!`0T34l!zLSl3M$OT#+SL> zQbj-b4ayuT00A6Ciw(NtzMrBvyT*R*S4u=e#LS?!l#rD2w!7xdz}(kqcl|c&q&mH3 zoyeDFj)BI0r_Xbfg3n^MqJDOhunaws?b$Esreukvj!e6+_9_j~OXvhUX+rB7@fX45 zHh|^_4>AbJ`!}*N;{J*}lF>b4y6rGmO7&6eFJ?C4#cosRCs-f!5408<4ikaF%~@xT zQO-;>8m6dsNGu@6*wDArP>h>`Ia=`y#3tx+_Po||{7k&4;H2)}vi-0@C)}Yx!FsLh z`Bu6^Z+}q)??==gDE4{u`T-5o8?|j*H*gXN5Bv%>;!mI)FYKFM(>0^HjOC3(w6+>O z4ksdcH62}gNF(%tbSFykgLZ+!`a`P-^pcgMn!P(uVe97f>=MhqW6N4@P^m|hKaM(* z*~3aCTCN5+nJ8l2CHy)*03N(x0tzFRtbe425x4x_H`B3+y6Ts+6EuR?iHJwXx8gwV zWRlMUyUyxP1i(8k)Dfznv5vS$Qh*1R0q35D0Sew@ZeTM1T+jNZjL?1M%fvQ0Rt=-p z8zYIfF)BE=q?C0;jS*|ave05&0?RwyQgVRfg-5xFG+f4&U}Hwxf*y8uDHdeOE7>Q} z3Aew(6{c*9yO9TKKjJe~IBYdPO!jD8&H>5mn!<0a4`C~YMv8hibH{(a(30Pf0aO75 zm)Oh=h$M=Tyavb26X3Wrw6kzGro#*5Y-x)PYs}Cypg)@!6|E4|k{1%s-UfB0r<5~N?k5A=Q>nes3lQSZVzNTx{ zhL+ssiIg~b;}C%wm&s@B^d>bu=htmBo2RPOG*|Lb`dhar{Y@gyZ4Uz0kAIIq%NwVk zKe64I-LMp_zvo-@^R0$Si>Ae;{}-)u|D394diuG{vl@TjI^N-rSM+=U1x6YKIQhm` zPbr%`7Ix}oP+NNmBJt7Lxi=uN%QCBd z8%AA7V}qUB1@;4nqJD$({lik&{y(<KljY8e#zHngNuQkY?x(0YwxPkQR~d zuAwCbq(c!!l$fDAhb~2gp&JB-?*3hT@8@~$_mB4<*J76II_K=O^X$Dp8y7zFfe&aU zPU;zpQI}t#;}ozyO3{{;yK?GXB zW%X%jsNw_<#bNK76IrP+o_-xWj`D>kt;Vuiix*Jg7f1s*CO*qjbh0g6; zS>l=#;k}eb2UN6>bOs8ZSCE63w?w#ALnd&xKxNK$H6IoWQcC<}8vo8P)|vebZ}F{d zbzqCzFyLpxr(}v)lspI*mq+#l6*3YQp`I!R#AwT~q?}NEJy#$TmJp({LhK&a8`0b2 zx7g;$E=?~VZu^}|%V>e0sK5Yx+l24`HJHc&AkqBCDl_3BFb7A1?Jnq1O&SO^=WVDnk68xAQ@6%?R!o}v# z353VSyG@s`U)KGD)!$70Ts>R%%Q(^)u&IdYfHWk%1zo=EHK8lnEi;vu`|)29M}eI4 zEYfgJtfS-hTUlD9*ua1zZdNZPi<{pZ&C>~z8HfHz(}V7!Et}COyU~guRo~6gw9)Tg z=QtBZ61+IS70i{@(ty;sPT`QWRFw|g#P=QU2D(XKG;937ijEM$e@S*_3h(eQ^>1S_sJDD>g9_oDg zBCKJt5N=M-1npuaIsa(2CL|j$z?!*G3t*HL!MTb;S@vzvioBUnvBESUTgu>2H6ZTTV*J$7`B7z;TelIsW<6Zz6;BF+%L<>a= zg15k$AQv zif?o9_@n|tHOp=BFQ%Sjr zlOu=)82-a(Oc1gs$?^d@%MFfmsG-Gy$BUvx^mzTVS~$lwU-t&krBqWp^E4^DUJB1G zVP?uN%EALJoJ z$Xo>Ne;G&1wG;SIw6{pbNgMEJ@N#lJZfuTk#%{D?^8nBcUYIZX?R?$ql^Y6x7*#}3 zhyEGM*`VA2WUi2M{DKBM7}B;r@Y|}V?=&_Y{TBXAcPIDDaudb8|E;q4*I#GIFOFHC z%@Y7TcI4v+%`z_^dE4|ytmkv(Qf#Jv$y-^u#~`_4$z->8fO4%cUu5?^DE=^d=rfv; z2d&oeK=5Hwp3Ubxpw5u1?T`^E7BQMVDfHS={ckt3hsK3kBeStKbd zce8))TZ)1Qn-=BgC4UNRKrb)0oY!)qO1rTiJSE@s_;e)llqROTuWwg10>5>{{N80g zGBB~}5P+CIC>0w;C+&LY&KrJBUDuWRB3GtyNO71a{O^ zDm$1K$3A|Roxkb0%MSpPgPVBzDcjkSvrXKU=v_wLA-8xva7=DJU3-jO9jtM4x-+^u zpk<h?6@*9e5y;Je{-;N@>`t?tM9>5uTiFmBXg zb}t^0g^Vn8W#5`j+T$%b4j%xcF&cBZ?-XSoyw^@xIw8e6@52J4k`v0+T;PUUpS$c3 zv7Jg=GtQkR8QdA0XN8WT=5Rz<_>i^{7KMhs#{FL5Q42h3MTqoW2c!1r9DwnV#;r^Q zouw`mv`&?vletd(hFBo7Y}$m9;{K79k~x1wB0TJw3M5a>c-s?v_i1;c_@hc5R8!?) zz_BZ@DfmB!)%6y?HLRzwct>9K@Ia{Pe?+#b0Mk*TPp?!$bxaGr}uvQav9 z)S96YgcpC5%}oZOaIoA*D$Jy1gr!GhQlJG9L&ln)tZb^`g`zy3lEo!@ww-~^qNPJL z)WLY(L{`J05ZMe4T0q*4BBV&GzqL(G>U=lQIBNg;-i?w4IPM??rD&l#MB*4YRn^9G{#fBsU)+xzw`WhNy~KIpAcN;RTWHR)P z7JA{=JDWe-us&$KtS`|XNw$m33AB6hp5`7a>!`_;s6Kpr#Qhc3MVW$F1p``hdshkbdx(+fe zG_h=#i&CI>KjW#N*#}3BT^|exmKF(}_wQ?+qyhd>v~jeK+ja_Tw5THZ!HPl@2L7Gh zQ&(u@sWZ&?S_cL&Nsh0A)`RYULsvmPIhQ_dx;f-Mk&y z0ZMC#4pdLds>8T|hd+)@evU*2{2m-Whx@)caLKtPNxA}-iU9lQE!;Aj==@+R5SQG= zflDF)D|3_BT!~d%fu1`{-r1p5R7Ds+eQE&iazZx3%3O!+%^GoqpE#%*aN<#uV(>O3 zU7|ZbXWw46T;?zLoE%M_Z+qtv>SCPVjQiORVFKs8-;lxH6Ou9BD=bmMvMiDgZ{|8U zbaGOu!472DQN@Px&j{p;e~+Lthy*nEbm+yPt zID7zwNmBt9EsYXH37k^8SO`aqDfwfN>WY5Ig7Aof3)0&bP7CaI0?Op8tDNzWDgjNS zHm~mVvxPa5HA@Tk_ecvvx#gg{=5zNx%m*%* z{0Rw{b$NwQJTA+lR{9e(hIQPzV!8F5vmDWZ$cTM~U;h@u`Mbl*!5d>7*R)%raAiqa z2JN+b%cua3GRj?_$QsE$C9R4MjQ|ypWz?U0=w(5*#Om05Z6KMA3ynM#SQBHHEYNm< zKFn|ZLcnZ9QQXrrTvH z&?RVk3!1UbTb>Oh@b{Vz#9dz-F*84v-TM@Oy=lLk5aBf2@am&dc*N5$^2E_sX%CI@ zyty4lU6;wGh;<3`{1p+ti_PA1Bmysj)@R#yCWFTPZ?A7YRP5!p?@m6`lecf&si@2i z-kB6fnGRoFFgX*XYv%`8wvtV)5u!domE~M^6$y>ay!(#oT<7sUC5e&qEwZ!z0op&S zeyfT1W#O zrLXp;bIrK%+h5Fu)vG#Cq6p0K7yQ1xWADWI3BR1!&o>jSe`gx1{>DUT5V(ZrWC;gX zH=pg}FiAUo*sk$)gw^V$K!P|{3QQ<*coeagLQGLD3ehy`k>~46*B>ctF_t#9o|yXu z`#XFew_V*n^8>FzN<%NmaG!qeYM(Ybe&dOI(9|TCl$e!!k`z%G6q0JWF3Ratg0&EV z-%BrG5wCN)Ol%qr_4>-LtMFaqFqAMaNWJm&Z_zyQ^uGvvOZcVG9#otl;Cjz6VEG3BV6oDl^&my&!h(w4xSpRD ziv1|(h!fpk=bld>jO~OmprDI346rt3f1pXH()eYzn94Hk4;gqoC|$mg12!{RN=hCF zJ5dPgCIO+yR9Xbl8Z2HDVPaggTFQ=9WwU3ZMO}p zgI-wRgd9J-0jeB{cNi{7aOx6ju$n-%6eZO~%ry=P8nIbF8n7eOrHpXIkr-zb@Oi_W zxxRuLDa1KXL|_HYK--y#%0VSIO~U~X9%%Z~`rB%iv<{l>MWSqFzv?<*i|9TOdL!pa zx%^&1!F2o`9$~Mynb81cbiGKFPguY@-{FT;-6r{81UPOEQ@m@!%i0bix1dJB&eLa% z016!`G_FqNA4H77oy(*N9kZuBvq){XEXmR}g`HPe8vC$tq*Hy3-|^4{bHQL-R3Ptf zfz98!G0i(wwh6+9CI#S~nLJ1LHH{iQ@;$h>CBJ~5Eo)oT$h2rQ-PjZ94MHMU<-O-v z0@D1Jkm?RrTqHeZfk%gv>t=X#0x7SFkmk?4c^{g%&Dj1;(Ua|b*Qldf-g1yk=65XS ze|#(vc1eH1{A)V!+XXD1^SJH_BQhj5dqig0!<*5eu32)=26mLx(w+-itU15=-IGt0 zC$;Ev>N`?h>vzn{9xT>2N6zjO~FMx~RpqeSc?hTVfdDib={3Z?KkwLYnJchLOo%rwX z(%GiaH4cIn&@Y5yptyW#@9Jh5zt8bkcE!}R?{ldB^;*)!Z|am~wi_mxhrZNgv`CB&%sMu-NVlC1vUJwi5dDEgN`A;Y#s zaE1cE!MbRgPN_8wa^4 z8kyHhW>EJu?euB%S~@6Vpl`rYTE-fc%9{^*1E)Cg8=lb?zs36`c)tRT@)k58F3q^FWu1F4g*JoO~ z$}a3wZ!TR=cG+QTVZ5D-)gLVfR3f-cim~40w_g8;Ea%Dhow?3f-uim??eSX|XOxfs(Pu(< zD>iSVM|q|ZCx5EYJ3KaLAKo|a2QQ|x&G->?YQxp>#s;;Gx{mq#jfHgepyK&kzt6H~ zP6^eXMM7oL71i5y8}fp6qi3K{4<+*@{&bN4nF-U4a?5tJjcnrzB5l0M;!SM8MtHeH z-)lSw(b<<7I*p;^=-0OnGOfv@;lY5Uw+-AXBAh-_({VYT&|$WZV8@)t=IVkG zenePcw=?{g6uyu7gyBMuc-+I5tQcBvlT9-t&qYtAV0d)u5XOR*3m=J>qlOjE$ zCoOPUNu?4vgKL71(km*V)l^X7qvdu+msY`6z@h>hb3On zM*yLeB5;j@^MMk9oWd(5w?jfH;o&W`N)DW>=Af1ay@0z+aIV=v+hVJ$gi^|37XWL= z(JjYZqqwbwxI(}zGLw(}!Tm>F*NkXo{_Z?H9tY-TM4LTYGatRt7mNpU33pJFv>Nua z2=&3WMKu^%gkBIip*Y+_k?aqkuYFqEgZ0&fSqFKRegl#onPKT<0Xn4A&kAC&U1Q%@ zbO=-Z1|%Ma8nDEP3le8Ka`7e33FUF;aXwmO$yrJ-Z6Laeh}xrwK7z8 z^CsJ?=5nXmKTIt5q}+R{$KdpQCplICQ95IL4`YF=8%4e?QAf|%Xgk!8{CVS9dMT4? zf1U5!|0?A{pX0rLRL9wVX3{Cs%`lJsWN&k3+Ge@590r@4#K-q}Nd;rk`f_X=fBMAr z$KUlQOTAL#eWo5)3-!4ro++FA+njx+4`1)-Lj%R;clX1fW@+pcv~Zrsf98n0>zkI={cP8b2C_4Y!EmVVRSJc;_Gy97_~Jv zm6fmQuV4LJH^?U;#$h`M+Yb#{Ol0Y0?$wEwB~sJqbNNi6j~&k-@&JMZ&4;={UPEgj zo0M2u%0GN8LQR{%Ljg&)gnV9mvMLXXg~2=!zH3`22b-RUoZKBajV3YWXq$L2mMtO`KX!tBvmgw{VdSwz7{D2I34@v>b) zi3A@TEiecH)dPgR{thA0kw#%n@G}B74JSRV!*x>kTcEs*tk`+qxOv9Lh%M~;ymsEI%~LS`%_4Wvom31UCvAKHIv((#3{Sl{(vg0;(x8@^Sooe zW&wf6;>eaYn|?jWplZzl!yS$-@12mktT?$AC;g+qK+`jC!DEY3sxyi^6~^)b6_I}C zU1wC!#t5~eJxE-!C@JL7A7X$6DDE`0u{}u&hZXgxl`yzF?D)NB5B2$&RUCu8C9xvG z!3!gIS>#!6vzjtyfwcz zk3Y1>+77`Mjz|pXQrpxgv|#*9#gF`0pGgXjMV@rP<%g)ZDuZZolpq&Gh??fhIdCmO z%Z8k~cf~V3slKVk5sTFO+G*157FDtb=w+`ESf9&&NB7D?5;Lzcfb<~ z0v}R^Zq)N_uRe9hasBMRm38tD--+;P*gI9Zd}2X<T6XMiL+YCJB9DKL-|QB=8D(Pu2S3F&hHOFJKs;pn%rfF%8--X$dW1T_XV$|+ z*$O6~4(VEJY;}+n@~uFGBe}6B@fUG3HFAZ{pBV+-HX;xTe&9{ypL>cS^8Ju1I|?O% z$8*X_$qD#o&|Ju-WuNdoVQ{KquQJ?{&zdF}p(qN?@^EQv1M$faewOIk9xh-F{-@Y` zAhW{ANzeHrkyRFFki@?8mK^;o%N`%;e^ zZGA6?ofX7~!rdn#M=R$K-X}8ZjuE!PK*HJ|QG@73#)HTgC^AGS0%m$H@Ylf0=Fw*( z7fb>rTU9Z8N`oTuM*h)Kd4#a>ZS;~Ajd5GH-yroGm3t)hBji@3bC{$}PPF0$=M6pI zIOS4OngcB1|6&1D7ZobypQ*N@e4e^0P7hMzsV#+_hmN4w>V%6jxEAP*WyH^Vr-p1q%axw?^~vVbDG{S-#f4bG9y~bK7K@UK;pq*A*5-1t zD1SC)fr z_@#wl{U4I@VDSlY$7g@?8hk|TlNPPl-ap^Md~v2j#|!ErU&Q~MydPDAy!Pfk4>LnW zzSyy*$C1}Hp~^8jJ4z4Os(b#PI5&OzfVkbl|t`F%pN z;v}$`KqIEY+*Ud`T7g+v{G7#D^{?Q-uB@->H2T#mHmGuOnCY12Dx20EA<@eK?zqRK zRvm{|t|&D8{*v~~?IHnd7L@l>PU||o7A}E*6VyExqw*({==@@3Q<9D_x`&qe+AAaI z1*;@s!G32;+}~gM(WBz-1g(4JoDN#{+Vm8H{4QDHrb|g}gN{?|1n;R9I~-Hj{Fgul zVf(xTIR&g|a5gkegtMK4#)R$Ohx)L5Il1?hH2fLaU0q?$DQBUQf_67imsS`CE;jGQ ziT9Uz^>GvdHUgY}a!VXuRE9@}Q)01xXT1T1`h%IQ%}TzEVHWC2vBds(KdBn z4ZFX$vv4}|CSr2DEYSGp^U4={viU#BWsl5Zg}aLQTR=k1W772WO2VjBpz(~|TkEGi z7FEH<_uu*GUlyH?tM5NzXYSHUo=VJ-{#=&Jb}bg+q7G2}v>*_%JJUCW>=TmTnkPQzMJTEc;x;!>sTCC@0S%0F*@>Qa94eq0f9suI zR^Z#5%Jd=wkwo%^gqoI0$K-2=-Z2kea*RR=V&TuD`+o?I@8eZaViSWVB{vUV65Wf( z!ofLZjbpfcp5#Qto`1Tc1M{x9KArXs(-HV%LNf4~jf?{I`}$G=nIWpNxO=`9ep9)%tzJc?nK2IUQ>2a3wV=i&=g7=+XgZ@ zB7#cW^{G)C?b1*f<-|7~`{{Dq5e0+2{GkacO7CQQ{}LD_%yp>{p5l@-=CH0Cv%NgZ zQSHXA@{Nudlqm<-tO{N#Srs=&JNS>xYX0gX2920acKvuxribKYq%JB02H84y_N-2d zjp$U+lKq^VB&@0E^t!#waL;LYDxw4_BMLa|vGhOr!pwBp!wy3cTTJBtYX9+~4^vNX z7&4Xt=H|koqU|R(p*2WBB;kkg(^b*}WO@lZa+B#I(lFB4$NKIMP5MUULV0j?4sxi5 z#sBS?ec7g2Q$efy25K&adCJ?uG`SR>Rv?Lj(pW4?|$|8QyDATx<88hCb_#D>ktqt=lCp+|b1GGTMW)a1mJARN|*X0k8 zwLr2o7*jVQ%?Nc*i;1wIPZaTbKg887fQos0pXFdW$~+vPo?%|Y516hyF8xnuy&YG2 zKPZxL3XDGFJ@s6yUOqKmS~{6_rZ||q>q=B~@@(bf4Ql>=ADU!JBm?w}wX=X{kcbhv z%T!*rATKDVbA*B=I0=zNZ}RsC7!0_2nRhgMFvT!8!S*L@QY2xJc|S1EbL+|U>xKsH zNdF`GLv^WO0e`;&v0}r&x|6LDxd^>OjkEh)1+i`G^6P}`^QS3kXbWAVGoXOg0EiW% z$$|SBb>{C$U|}*D>lc0}5Voo`Y;eyl0+_A5)xPr`Xw4@`_n@uK=A{w;p$AQ}BO+y$ zL4TFcpqhEv@paPV^-p%ZaD^pGV3$5|f5ZOU9swE17FRI3JH%GWD*4*NdmFxDKy-JO1%NCvsjA^N6F%X0B zl|MQEUjh(?MzaIjssNX-_Acz_#pC3+{_A2mILVgtX|H#I;p8yYrp7qk&+E26mv`Je zYmD`9i@kBNtFOju+Yjb(`ps+qbLk({qf96siTXlk`MPnix246&)pM`0Je}kWs@cJXM=p=w#Wb-He(pS|b+UYCvT$~o4&54fU|K}zESpYu;R#)*bUrhskdKm#+ zRte+IdnuSxho%2nZl>bJ#~stH=~-JReI&u3!V~m7-J}lS4XpkgpHyrVt>8;Ec$F#2 zz?k_I4@bFsm_i>qG-BqczWeo=xN2LW87TA-pqD=nCKccD0zesHMF?L1-7;?N-?Z+2 z2(3~AZs8S~x*yZwP<||?z19|OogQRU{%~u0yU6pgbC|Z>?8MG+wDIe_cd=<@$T^sJ zjeX)r#1pN5M@H|#K9>W)HJ}}gD;#tn{5gT2M!mnj(Y;$+)vDbgx9sm`}14z*-i?=UxW-i?u7 zR;_Q(XkywwP)D%ub+5Z``*QJ@xD+`gZ2unA-JPxYwSJ_1cd|&}n-z;+i~cXa-D#WG z0EJWOr2bOh5ulLbPH=7cN9U4-TOU^V&*eBB(w0?R*Em~CFwe_vI|{^y&ez}5^KKF! zdNX8fyPZfik@{`8W1XdVkc@^P>G;Z8$@P_@lF!!8c2?tE|lCUtuEkH zPh3g={{OO+Rv19|(NBf8|DAN>?rM)%^Q$VHjwnFe35G?JS=C#Zq}gn=YE+db-(|o3 z;GMF;-PwuGFXFGxl_<5RF#T2Eq*I&+)*!(d2@FRFO>a&VTVDH#*K#yU=xDWWst~EL z0IKRKo5eZT{~<7NouIqxwBfXly5sLpSHK-`%X_^vJY3#A#K@OFu8il+WLbxbb_YxTx4KHDRw z%s40Nxt=?V!x3gXg_ftC9A}z9+6vC|F9!a=g+8Ig%cFlZ@S0hw=apF58!nhd=$JJx zaFYhiz&UApo;>_dTK|WaK8fBPjZDi=?M{Z1=I%_z?!G49R%Bpf8!#pJ{omr}T?zAz znYm47u~il2P2E>gftj*9(NiNct7E3qe=F$9cJ$}I8R!vl;)kl|g1cEjEXHF*YZlJl znGNrOqtK`WN9hta0R2B+3ITJI0cX_;OU|SGPj)2ki0vOR+TZZmY^0Xyb&jRZ=JRUm zQ^y@t#G0LrnQwfikbAxC@eM!IxpJQ4__wrXw{6;0N3G*<1%0s3zg&h;1j08v3;1{b zgo-v9r%5_S2KXksJj(6K|L7**v>Ma^%?tY!dP(mjNdX^FBu}vSDoQsA&4y~X^79=Y zGCr?-He-iB3RV65IH!6hk0HxSc{V0a=hZku!s#C^e(j2KIAz_f4i|r1j-zo+xnt4J ze8(+slE{jT?Y0{pXM$6q=o@x9u ze-rZz+_hWY ztlZnIYUdZ7!?5okkKgwK6GYq%r?dr12Y==dS9=Zpm>!sLu<(5MB4O;TFY_M+;jsG3 zf1#A&BrA3zSG?375E2`hXE75C;c5&(YiZ;s`u{ySdm?c1UJ4}h-PS;NC&v$Crl3oQ zJvwyc;-9&mI)a$~0`y5uAbave;9Ap1@n2s*`uZ9;xiBuD^GVdIdH(Fi6|aWkEhV%q z+1USQ*_iT2DorKIZEcx$R<5HQz1LS?9V>&=rcD0mGTC z{RH(@k46$R934%vifn;UD@)=#E%E${$`QENxT2dCm~yK$zhgXBS!wdBqEw=oT2_7a z16Ttar@p>kJn-TOo@f8}hmC222hW+*%X#2-Xh%2I|MzxifZNfJXu6=e6HgQca69{* zk47Ipd{r|(cNO4-c;=V#LTtOAV`@quVJuE5Go`lwwYQzkoxqgNc8%$oRN1+>K}lNr zqW(dJ1cR$(uZAD{dt+p#D~)ksmu)Y`tCdUtN%1CzE0EmLALrNKum39&9|3OZ?YBt( z1FWq=3am}Z^h>3_Q(!|vF55nlJkKuIc%sTa!OnKtJ3;N@V?D@_u%M3`@Gr;Xe@=I< zd)Fj9wC5zzs^-Jv4R6EX|HORoww+2UU!R^c7DKpS)Km;jX&8M6fB(Uccf$sJo+QFU z9U?D34Qa<>oFUY%E4hd_ZIvjipR6nRLGHm1$5C)ai>qZh0Dv`K=W4~l zVxdanxVE~Af14$0XGz8=%KbdMQ1uR8?V%B(?Z=DKcRJbPWG!jkw7oQ&za`|x%1j$Y z|2hCzZQ{W<0mev6)dhL_RBy+khX~L_Pu}yNF91h>OWQ{8tJcR709CJhLWLHPKovvB z|9to4JFC00kh9-BXjV04N}Fx3*OG6QT~ll~!7?#dvqK+$#CN9lilKMvMf{J+z;y@9 z?AnL3QEhQ-Mp8+xM1wBgaG6cZ<4$KfqY+0&bbX+)F_-sy4OM{G-{1L-OY}b?Q0!P6 zUIPzn(Iqb{F%<&>l>mxN5bgo@T(e%T^xxNk$ImYTq%aHs3y0_=hw;yuT6_5aH-vDf zQ4oRwK$Dsj{%O`In+0E2AMei3Z%&ju7v=f=aBMkW>l24MUWP3MZ&y#i&*9wTMSA(+ zQw?u_7eH8VfE!VGJ7t?62>5Oxl9q!K^8r>s<&i4{OJP$vnj>`^z*A_?`1o<=4Rdz_ zi~ZdgW8m2yk5k@JX&gXxn)aG;Rp>fDWZZi^ebad~`QPTj*i1*G$ytgu%qIIKv zi2O}mWV~^rdsF%R1X<(wWTL@&W-VV(?MwoAQIM<4;SH&>qT3xec4nDixqb??abJQ{ zTCNVX?Z>`N z!0Ds63L0rD1waWQI7eF(f{+butNodJcU*WdfNrOjo$XMe>EdQSoFmoV$EOR%wBFu~ ze^F0dzyluTY-3I-VhGNis{y8MY@q8(F&9aiW--nm7^Hp`nl4cC63cI zp%Co8M8WN;>dG*3Y3{wQ(;h>WL;%3BFX(E&XMs~OTig+65r^$e-PnJ-+7PlydtU&; z-vR*7bg$C|C@3hJV5Vz#3G7*FpyrTptAIgzlXXpRy9<*-Zk_2LAA!2+*w z-wfbh67G=#bMepMBa^_#@utMQG?jNP*b< zp+|QPg>zfc+QW+Umdl}_T#;6jT=AdJ22OTo<}N%m`8S>lEK?b2b|_7@-+%s#kSzcV z4#T{dbF7{1pVOceE8yklH?A23ph>LCn+~)TPE5ULJ-_PWKf^Ovy%FQFg9J$VVkyQx z{5sn%VDWhd!S$8B@C=YfYA_~8*W9UMfDMkrp2Kni9tP0GY#?AnN^6}`1Dn^5Jb)U~ z&V%23Tv{{5`<5E#5&#<^>-E%CPSQ%4J(EiVkIwn;%3cCRH;&VewPd4j6fqnc!x_Sz z1VBY_&Ub~C5KNCQNf9HM@DB8I+M#mDVYm5|JRJYE^5DYhk#<%U0GZ|!=8LjNiHSpG zp5%MQ_^!>~Jh{aA0At(Dp+%W?XbSuT5`R>P4tmg+$4G6X}=g=>Rk~zAK!aK4Kfos7# zBnn*_6dB>Ki;759Kb`Y0zbkcbzZQ$mi^j@gGjautEp1Nn0>Cjdfv2ScV8qGS?!B1(lfPy1 zEn(aBHrE+NM_<4i6nH_o6=SoWj|Swf@6VIZeMOr3yk)X~)%wnlo{KMBA@VLx_QlQM zwFu44HTkFI2XfP>bp}jZ5!6A0rrqf&38(0Jb+PizyJ*2)CVh#dA7NJu7@_+SVS4pQ z#jx93T$rM}n;Sg0d?vB2?5WqsF21o%A!BU?>Lz?4n1_r6US}rQtp^weaQ_dN77-7_EP0HN(K6ALoPf z$N`IfpOA6mX&W96En7WN8q1IcTQOaW)gp>o4?tVwkMZoTw&;v(FWW}@Ey`V=n&g3= zzRzD!n?CEm#zS=iLqzGS*b-Q}fehq}z4UV>5(clZKi(D*(&*j6(+SZfq=Dqu{HJ8DoTG_rI7 zic!Eu5+lhNT6$RhuarwdW+DO#^mApp^Pgaw-?IK*M4uNmZ9l*;s@gd=-ZKjBbY#?(wAI#>VN$ClmBS=>2p8t4%SC@d{slTm zI22A0^!tIz!{y!t7d#2Q2OI>#>+Q+sA)jak%m;Yr*FY+wc;b!`5%}G3;m;4{6Hq(_ zpmf5NlOF1>#hS`h|L#yxjglDf4tR1=$pp+HG3)URyajj%hVAI6d{A(rM*GwdI=SB# zXqzE#We(88dYUg)shF_YY>TPD#ZJOczzpFtP^p!Bf{l~ge{nHs$4 zWmc9Xp}Sfp$J3{!&PPw&=SQ&@8|4+W6GSx`t_o`>G zD~j3Qc2H)z#)_qJH%E6I;UwBSNCP~6|6@DEWZk5cYBvAyI;V8RV5M8CsS*`(;LPh7 zIq1PGEC$(PF}*uF+m>AXW2MKZWq&-XUWyZkKECq}4%B#`ci^xEK=Gg#P%3B=#6BRD zp0gCc*buEcu$2^yA!m^b@KKmU3^2eac1t3;qprgDLM1IvzmGVQO?GOA0wE9#;()#n z`W(T6v5p)J9mM?oX$TAiC!sL}I!EyChYP}*p+(RZi2meDP;u-w2K8-$&bJ@x2A~rh zEKphLksnH*B%Kh9k|FW+$&H&3Z9MoF3X*%U8)zQ%(>}ILg51U1Y0KD$?{zY9U-ljX z+C5NV#`O_Sz&`#c-J?0`GmJnFt;>nH60?%{Au~$G2o7^hcEht|L;~0-OH@j}@28sw zPmkh1wb|#O6#{sQe^C@dxcCti@Wcl2$-?7S^M0SoFBusHzvxu%ciK|mz#qZy&&cWtKj^X%NuwAGK#h4Z_j`i``Gr`=KFjy&EZu{W4MsEdx}&%; zft?X@Iw4d5i$>&!P=@W5Gamv7;EL+=gX)b-HA^=fD> z2a33t1@Oj)T`XQxx0V`|<;Dsrd$`Dd>33xb^Bei*)wk$a z`r=rN@=$+=D0B>xRNB!2YrV4Jn<&#H`FG4rS^4r$`6GlsqVA{zCtR3|1^t}N7k3eVpBOuw9y?S+n$b&LIS#p;Y5EEL z4bvYH07pkW4&{7fC~8gLXsS6tnVrSs^#rJ1jJlowt7qE#G8|vywF-4F=jcm?!Dqa@ z{0f^&qDYbK8?fjewk0hNZAU6K!BL;nRRLgE78s)CT|&_j84=83y+Et)r^Dq=^e;jH zSf2mPDt5j(L8CLGH0h)&ihITEXDRW-froLL?joBPqZz#;UzA(9PES)^CZN^cTK44 z4w{SQPJ+pdTr!FuJa)zZ|0sLwuqwB$eONjcvFJ{bPLYysqy?2lryv3XBHgiMAqXf4 zNTVPPi|&+C1Qh9RK^mm%H#*eT~2-q1S|+RENNS&j@DE$ z)Tw@JsACn*d&+j5gc|f*eLcXq$=RrtWQnS9!!JbjgJSh>4K+6h@6hmJ>V?Tw<3qX$ z{01nCVQJa zXB&>CgvS3ZYz~hKSJ5N6!j1f9_;fim0;;|d4HplW3P~m2W>&@x(u8*eCxts^Gur6X z>W14f=Rc`iPGF|M=D=j;XZhy0t0b#L=7H>eqj%R5jWvFH*hGdl#Rr=uh<*{;kxdy~ z%U93=KDGc@_>jG_Q|vahVb=q*k#0$8hcdIbvsoHESz6BR31+Dzqgm?Pj)(Mh_`I4p zEzzRdXj7~pxe{sySZH_Z>B0zlR6KCc+Sjus6vSBrzysreCt2=|y&F`!yBKXAdx3j% zgZp`Q2M4e!4G-mgX)bMXthw*6Z1?VRbIl)2(yT0;gy?mOYFp=GJ!tBVjp@hczC=WVWk}&`<&qF zSbA53OB>dTbJ^KXCAi46ALJdu39oJQ!_62%BjXS4GD8?EDQf$4D8iV{n3J1Y;R31L zEQ~DiO<&1;12G03D#~wIrB{@&@OGrnSZEqG;MpdG<6T1tWP7l(d(XB#u%et2M?>>KG~#j&;m}#VJrX*%=9}|} zpDI&~(2lr_=x945Jt#SdsD`z@0|hLz8hb!D60{?j{ZX}V8S7Ug3Fj;PQU3jSkdG|` z6%3N2RowzJLk0JufjjQM*pxY}@sr$sg(qKfT5(!gzIdqJzdKOL_>3ko0oU(Tzgha| zZnVFweFT^-aclU32YK^8Yd7b{%uVhFXwWrg>E{A~7Czt;|3)@km&IQ#DPBFEVVs>; zu2xP?u^(wnL2~UW;}i+K8V;Jg(rsoPA0NO|h0B5-!efG<8#MjO{_bPds2LqD1p+@- zeWS#VqfQ5Jab1sikc_n^R5DnO7J?8^TXG4O`}ys?s$4nhw-ZeBXH?REgn_DQHba)%)?o{h=HWtAQ+d|}6cxI{9202L9OJNE z9V@4tx^E&&(u#Bv3GprirC=QbYO-N4ISEh(MK6C7ud*Q%KGg)Eh9Rbz={o>tFNbU4 z$tC%dCDOTJvpT5ubRh7OEVkCqtuDxqQ#@|hqk&6$` zQ+URLMQGy*v}ch3tve!YtUUrZOnER?u2%kX(ZCUSoFH+P%Ao8Igt$)~m8h=?50aOC zPl}hj{DoDmNX_~#HXU;?&CAL_GFw(w(Jx%!g6|HD*|yd^^iU0s$&c%h8XzaPgY_+J zRITmaYowg1wh8jVt9C3)W|N&)qqSneyR1x7oBO^iq}B?_oE|FrvKvfJEZHY|smN>E z*@YpFgj9$JVVHU;Z`gR5pT$-%erwD9s-BGu%2YIAc>zYPN+UZ}&B2w2AC zgA&;(NZsD*t9*I&PLJh;3{s^0Bh^itng1-#3~{dmZIRR%wrJ>U7zrDwBMUi5JtA13 z$s!=RB}zf+AdpLjP}E+C{TbCRrJBmaMM26z32pZ0W*gxPBhkwTMo)|$8d&v#(`Vc7mKxDejah2UGY%azA`0;nJ_f5W9kLB?znol z`Evw*fl=Z@OlE!$f>c7)s*x`H2%+j{!#}fw#)`E0CfycCMjEv@<8u_+@2Z&&$|4Js zO*>I(_gxFt%Fh1)u4Kz0;Pw1_U)Tk2De~Gkjp(PRayIkku2`he-#KFwl<^?v zBw?`_-{_3RH_=F9THE~CI`ub}%ZIqsVa$xsWT-c_h6kCouo9anNnmVJ>OEWtq!Ldh z8ecBu@K%~a^hEP}t*u@27>!L@EPgOe)#s@m_@cTh;4|$INPdw0 zkC|jc4r3L~MLqcbHw(>^h?FYYT;G+yR_ixh)%%dU0eBD7)XoAJSax=R~6} z4G%BU$2d!oK|S*R2)@Djuuq(q%oazyk`B8Ww{&AWp-(LCnQ*i9$ntaB;#A9)LA4{v z4$B&WZx_Y}-(ceB@+W9GyWls|u~2vyH~1mDe#Q)Ogi7X?0t|I{Eq=U@k)efd$GT5Z zU6WY-zTomMEUgFJRC~yFMyUnFyWbw#%z6vi%d`;J0D-oAzRnxgEi;vYVkZfXL&tTC zt&leP{9$a!qM4z2!Ob#pOTkfLFodF=M25B2MFAs=I5-q|5~!5!m%vIx5=%78kQ3;@ zJ+&`fPuIVThWS`FEJnI=-4uBB8ZqRSy$UEH_SsBeJTQInd^o53sHx|hBwDB|H7jFQ zga`rP7d-xs@#Xpo;{u~7vq`ycMjTEfQwQm8$9yN1IX&WuNvmc7pH<4it0U4uLK2R$ zL_y*3J36W@jW{l^L)y>U(>!NrQWxg8$To1PR&cgeaWQ{&a&c?(vUI%KBP&ANz zvm|bd=}q(2vEp}S(t!cYy*3Z?lVF~AGoVdQUc{$dJZ(9-aMBb8_j;FK{d+_rA5~d- z>@rYm*5cZ6rqA@HoQXrAO1fdV#&=+1=c7)S1iAH&Ct1SSjjcV~M8Z)pv-#H4FRy2U z8u80InOV?g<6I7wB;kA?Oc7ou;nh-6u~kAtLlAiAoT8WMlA`^{c=^KsG(_GTeghOs z*UFUZab*Y;&lSGDeuurl;L20h;H0l0>fi5(zNl{i}B`w77KmgwUGOe@>ZrZ#-iY@7J z&F#*UyE#CWxTKlO$7!NAr*1$s;=gf-caese!LB)UWmdMMONNf$)l79kN58V(h6Noj zMPsSe?=V2J@!HJWxfo?}e=3b_ zCWbRW*}^g%86a5GDlS#hL0R@vuA57cUnEAWjv^AWh`rW7O|3kg-Ga}D**=?@a_>wNXs?ME4+~I;2R;a`c#@#w2Z1* z#rhC`OFSYJkkVtiB4Nvm?CCmfPi=b;1~7{j{_`{Bfj*y(dVM|_1%D0EN;N|8{Tiqz z?3x>hfFlRW1~`7&fnf)Q2b=h)rtE9fB=eGG+Ba$cgELm&y$kv?|5Lq5heB+RA8X6K z_NgVz=R2dM4d-hS)sGKuOM2C?dn~_l`aXPb{LM0MHUz)EU!pOTN|>j;gMYPpBxGu3f6?X*w}Gpd%L3aW@%&E zbCzKPKO7fMaL9_|uc&0n!RBY#j(%CDI5R6f=cI(dphG-;8d>qZ2Knpy07Wyyga3Gb*9OTHLiHcnRXmFdGvl+dSY`*if_Sb_4j zH3Pa7FK_IRH2S?FCX0~1Aq8@7i_1xJ-!YAtRMbjuz6oTT01&gXrkB-;2-dfQx~WdBxDUbqs=IK|kY8+y$LY zm;2$q!cj{KVKkrlXdg3uZZgxKOh@UFRDnS6z_O9FpInVQL`ZJ1bcvUBO&x5Odmu06 zI)u$|(X++g%gt2qlaLh^&P&=RRTPnQNISV21%J09KUHE|WFvL!ncbc@U+f_FY<0Q9 z5$f^9cS5eP-n>bC%xJ8}uyR4rQfeXAuCQ?-ICLFOO=T6zlz+8TSh>&b!mNFD2ZkVv zef+O!Oa)ZK5_+!r^R(BTYMe~bcA_JdqIy7-aAQPYbm~*$on~^|^_VT9h{E(V8;!)$ zDj7+!a)%fv^D*nwG^m;cy0f3msM*tYh0E6>!AO_KrG##EAExAJz?(^pdW`8B@8v4A z>mwMRHYuVjnkhsnU@Po(PzvxO7ir658PX9Xq)`>;4@W7$Fs#rGHODXPu=HcB5o{byaD7v94ZX>2d+8l?C$x5>jH@%Zh60CNR zAKQaN^l5W<7n3}jmP?E|nPqV-i!fUG4baL;9GV_}6Fr1WU^^Fo`0Z?F8#>ro99itI z*e!%Oxtchx;!njH`O3S1_e1JlM<6cWMgjXnVs{$P^Y^lWAwdkQu~DEIUg88Y3V97^ z2rf>nEkbBRd?gGQrszY;gJ8xrgno$4j7>kD`~F@~rSiue@17?}Y&V1-!ka%pD-3yU z7hw*~80(w99(!(q?Li(mfDW@DUdkC5=jgt${6Lha{PX-ZPe+6bW)Lf3wKERU@?qKLW$&tJkaUu0dB zk69TxV}BOihn{rsu?lj;Q*&7L=e8``DKTsr%=zMD%37U#N| zs|f3|g9?0}Nez76MCG)1%uo!Z zU^35MiAawj7_u#ad*7n_2lWbd?hA40=RxQy^X(HKlY5MmGExWk$DU_sbb;XRb#VFV z@4-qo6pSdMQ|xAO;H5^h^i~+$zK+MKWs`QI<;)!s&;liA z)Z#gvj1HU*W*a7newNcmaT;E2QC>j=1Q19zydI1;gbIBi)-Ga(x~M--n-~g4W=nT9 zcVA_^n-h6j@SUkl+1#>dcXB)LdD4eXC0xjHKau0szT7p%sGnc66y+{&b{&WC;acPB zMe)d3(e1*IF|g00eZ`n%xsFwRxYVrA?MilI>g4?f*=+=rWT>W3(dB!IoE+r^t%z9m zzLrp#omb44E>loG!Wg7`N7KNZv_^%Vl2Mij`#9PjhNVO^g0Uj&-bNfkACf%Q7T{4z zR=7b@Ct$l42K@!G<>pvo~ zKygkYOGCB7y9);(e6_Fa{;qp3gzTb_7`{nI9VF`vvT-Z?qFOZT*Vi$(Ia1r_ZS;?u zg1w`IsH@{-NYtP@&{X6bpURg(@1ow*)zAzb2Oswds5&_Z&n!Pcj&~ zu4U+z_P(TLHrU&D9!`br_ELn40`xk#bqz9p_d1SIy^dxtmFg;;0Ha4SjbgnkyfFJF zfX5hRU@B&~XsK00&yw7#*;oYF+pI4BG-xy#4qZsZYecO6ty99DrR-AO*L z3?n06E7p0ePu8RUDLzJdVy^~-415%2OzX@3#=UUn=b0h8s!+>B{%krWm}?GeJ9<(u zLkEqdcUfXa8QvfvDc$6_A<^63ZLT3j@x5G1X2EeToqmriO;=GxR<|-SSM+&R-zXuc zImw0qLrGAC8({X33wq6u^YnFXNY9cCv6|iBnOt^-J{|hkN&IoOKm?>&Hb$f&Qi38x za3P{g31q~tRt4UE2z0TwyJermJhuBvd~S%?-U{l05VE><1;rIFz80Zvy3ORy?0@Gj z){_oQr?NBpt0Yz2`|~h#GDjAVCcceBoU5@D%4ff>uI?+rS_2)!ve_>qD9(e#ZB->* z+djnsjkF>UCXu4nSfTR}woTQR*Ti2Wt(Hh>wwp5ij$qNQV75JJH?efG(I#`AuTyVi zA%CieXrO5wqHzpp5I0`t{YL0q|V>ZR?!z4}} z&8mX^jZDlQ%mFuN^TSu+f(YXI?-DZ)wq@68_NjUO{H{B?;3E!L8`DqfrNV<+Q{~1o zbtbH}x*mj)bzlb&QJr3Mz>Fu-=2dcx_Gdr$d)JbPjC~X~-2A(~yNeoom+E@8I={$7 zIq@;n4mR^nWW+AF%VYT4xBAezn*8344~ZYfeoawScP9{TYzcF#+*7jpx~ycxx&w0E z!xvimCC2XA#6k*NPTKjLMtN4G4Q>?v2f{gBC9WxFU1#huFVNI zf5+hvJC+WJAHkVG$H14|v%YiNtXt*ugXtw4($bCX2$x&(_P`Vlkb+c?G4dZ{XQ{Ld zh107bIJJU{J{q{>5Oxs6&{6OEy5KnI9DXmUA=`c7)3~xOuNx|R&^l?YA$|H$%q-Wu z=K(@?;s_QVA(x&*?2;XTr}Xx3E&%^?VycPvgiOL?B#}DD-d9aWbb6_$V-ZN~uI7NT z2VG%-9cW8DV>?Au6n+hU2ST3>tRimD$e!Yzb?^VeU>s|co#>=YKd9a@aojCia{XZD z$nu<8e_|n{CQ;_XuT8}fD{#XmU6|^GVK!9KV@I+pIpD1#xuB)rqsSk}s*=qaFyHb` zJ0?Y$fTXiIjDRRrOY5%2Yj#XxWkLhuf&$0;6g&oSO&6k`A4|a=16@bl}N+r!tQ$e{(vNBjESY>{#R)(^?HurYy2f zDWj2yKcDxwHy^Ts!EuVhilSNw`_MV*sFwHM=3JXU){%*Hbz+qV^XB9S(vnZSzAuLQ zSu#Oi6y(n4;-FveS{_#fa3KIH*vK@coqQOsssTIppSanu`6mAmjFeWduN-UyyGiEY z?362jkHPP3d>t2k^ozdGtXPkkm%d8%dlCm2fPv0yJfYOA3`IH!!p|R+zGBJmRZ#sjl)fxS5bNyls`YJa*_k_TTJsm(Df^ z*k2sFw(tKkp71$VH9v->*AK<0wgoU%bdC?Wo$psCeAq6x(l;_z|L$p+zPqdaCElYX ze3H1^)UZo-N=4Lfdok6^~UZ&VNJo(xg0;BSmYtT@Vsi6+&m&gVz>H!o}KFl893ef1@e zG6xx$2FRf(Y_w;KOkY4!Wk>ZUtUJB_S%s#MsfR5MdQ6kpce68_5|ILfPHp%6*wZc0 zo9|VS+hT`;wec{;KBgofW9KnA_x}Y|WZpzInLHF(Lw{?`Lltm$Z&`Z#@d2u{=4>X} za=5_JMs;g^sDE|(rk>onh+EyoQQz9(=5gmpvA>h=*5|(K%NeN=MCim#!r#rVZjBn< zDp?7GCv5e{seE(QPR9)9-V#Ia<3SJ?q=TGOkJ$awGB$^fc8)mkQ!hoHW>1+KK zeLXYy1ydE)zD;T=+w|)*cCby_HycS!$qiFmecz`R%HN7{+0{w!9BQ8J)%Z;oY4Oo{ zX(g76Wah%BBFf2pQ1lcqiMneBc6DjT7SH$V|>7 zYUc!O`J8@Xq$ zDd>HO@B3@|EoxSGr0k)V@5L$K>!5;qDEdRyx%AQ8Nz@2kfAM4N{8*nS9F|!doCVLJ zoG6SPeDU4v@7ls&R*FK^A;jK2)$8?P7!JtY+?yh95>Z)L^fms#J< z!I*bZ>9_6R(lRy)gmj5^o5}ZUJfHl0F!cVk39810?pd$VHT$f+opST>xP6n+gqg9J zTE6s86I}37ZGd5<)j9!4%D*tt*}~v_gWk_`&oksvN^}*5W!723 z!S$syz@eKDm>Cy36A}{*ER==|?iWAoU7F>p1X#D;fdS&dty@o~YFioofA^q%rwF*g zd8m8mzRo2JcGzPJq+qkZnmBlLu-=FLvcIaqEX9_tCeJc=bGH3-Ya%!*7n>m8P#5&Zugj0XjgcE6`b0KkNV4< zg1vW8%7&7KAqqBH#26qU0PJ1~l~lY6*371J7%i?bP+7+I^Qsm-rdr%|NQ>pV2_08*OPHDhO<50m@`~(m#Ya|VZ=rno##Pu z#U#K@D>f%PH#EfI=N>(_oOCi0TX!Y7roj9drEU|5&rE*{K!F*Ro|Pe1?S9+e3JPzw z-^IH|DPUg552a)b@caNkl^@bPb=P}LsU?u=e+BEP9RdRHWeQ1mBb?Wcc0EJHLo)=-wJ5W2V4We|3d8ZWq&z(cn~8 zmM}b4Z56HqR0J5oQ&T1>GX6fnLRF84%lS>~$K7A3^6v~=S*L=ja$QK>`+j%wAvjTJZeHBd}l zQUxE=-%jEZ^h6g z1Sy+febs=+Lc(q!U%=OnE-MKX;AaK$X>CM0R4 zH%)B?x;^(l)o4DLt}9C$&&{cFV7HVn1E@_DOMw?lzQ_Np46yttwD5~=)zaTLe-jPp z!COIfu4>nj^EZd!vymwOv?)1U<`C`VoMx=bbMg5d0DiuA^+vIs8C`(Rd2@wW+2-wG zz#_3Qrhi1oDrmX#)OTl-=J2HS#AEe|)`q61x3Fh$-v9OXzdt3Rv;{9sUlnRDWL^7_ zGefefL*g$KHnj0TtXZ)0=V*bWXIfdrW}~5dKZzo3jWk*6#imeljX~*1};=`;si>3vZg+GC(gg;E-LOY)MbCIY>pi^nPM+0@MzG-6dBpp6?J;dk%Z%-X0DE=7&}3 zlnFzBBg44U`|v2>j$KivpBi*e7-jls|MF3sD3RZ|mYw+9LDB?)D0!aAKJ z$J>^>8fs0?#WIH4l2siOf9(MVtNo1q&Cho1Mor0Yz^HcyZq_pPp<8BIPIJ>qy=MZq&M6CS`*Td%e=DwW3*-4RW+I*Ew$bbpS*OLGS8oePifWuMVKzTxZ@2vf=C^l! za)PBn6#Vtit@^W9a@Tg(KONbP69aacPiIxg3Svk5sse+{Znsl7xPfI9W5kJzKHtBD z|J(V+Q-JjzDr*ivP**aDLaMEavD@2yE7eb!aAWGLCXrnlPoMsYLq#`QWX83*QG}3p z7%8c7U8I^ECw^ITTZDU3IlQ|1`jI2MWfp5@!ILQxX46qrzb<1cRjQb(U4f_2OvK1d zXe%>co<$cMxWITcUg`f-LVT1GqR-|3i(E$w6}hrwyy~|-zvY^LaK+iJa@F|N$8o%M zOTvMy(CCJdM94_7v9s)LS)HbB;=;`^_&z}QE*?JNw`57f5QaF>9eh+k&H7#3bk9%o zmg|*+<>^U)9YDhG^XHPFYb5ZN!68(XuzOXH?>hh_QQN?$Q*64u{rzI?wA%o9yJ+KH z_-~vx2;FXhO|ua`cK;VzpBmg3DYnxy+Ld-#75YNl%Ld=@7S+6N`kAClO2=EbXqnsosQr$g3L8_aS77UUpp6?W z2>xvQqg7OE`(N6@`~qkrVj=G{2l!v*D3+aZ(S_xmqe)pRoQCY~`bcZvC(~cIq*tH*pGdPlv!N#X*BKa%%+2Q<;AcIZ<8n?jneE{~_TTOseB zSqXmd2CB(=7*!?p8}SXSGPW43*)#GgaTZwVU%_P`=E>ghb|kvxd_#nWV~Z2r1VgT; zrmjxDQ-Vf;PTi4b;YJ7MAamOGAj=(?+GbMT5h00}^A{=KfDZbU?&7Cn80iszdNI-v1mllN!jE93$Ec{8$dx1{Vn8rcI`v{YoM~)?fwG%GkgY)xgqC zy7A&%$5eGU(vc%3{-!T9M_2X^)5`s-Ti=HGr5j&{!NanOpZA8g8k>KqNCcE*TB-3h$rbW^1+Q!^mv zAO>TvgDCS~yM41<<9~cIf)>dCo#?wgKnxJ&qkU-n8Ba{prA2OaKV2!U45gnB z82+~Tc}&4p>M~LCmFp+6K*J}Ok0%4qus!x8=rmwGQCYpTJmxM(JOs394!~_e!rCINZ7=Ui>}`evcsRl$SL4^5fHuDxH;y(*s*O z#;-SPA=y3QzqzaaN`m#V{zvfuc@+ele>p_!&F_uaqhf?g+UQ@mY*)n~8xh2|nhvC;i3IBfmUZ9+@>*}h=?=#z?awOp{uAB!%G?KG;5Q%E5 zWh2QVP#kD+E{vau%j+j0F9{4NzmTwgyIE-{?Yr6tfT5&V)f0SFsNz8H%&A|=aM9iI z{-4u>AdRXu*~~+&21f-~E+)T>|KrRZbl}XpD)%;jKRB;Zv=~d)>8{d5jKTwqOgDQ+ zN{pGZ%_ItSZo#qmbDP&Xxd4MzPW2|c`$j~h7zlM-ztK8{GWpm&Z@TD|a?s6HkHya) z0NZ&dy|BkWj(S2x7k>Uudck#qobLg+ZkzRhG*w4@s-L*=Ro`#$U8ylXTl(mx4Lg)sFJF5ixp%jI|Vu7%q~X`-#4Ta zlZ&*z|MaLQJ3gE)v7|S;PX;d33mnjk?brSPu^}jW1gctQwR?XZb>G1@sOzQSwDV#L zRrvu52R}jirF(OF3NY*#KKH#>dm{X!xYE}w<4MNFYi9?SubUU}fOx{lTfiX)g zI_8tWhslH|kJH^94TYhUhch4K@r!g56`1Q$&U9-GPz(JQHBc|+Ce-_Bw#ee}Tcwv# z9^PBGxBNDStVeshuW+&JMZt)3#)?P0MmTxrO@~YMhO0gr(Yv8KW> zc=okY?Q-nkXJnM+*j^IQ{Ejr0aBlUektkg1h zV{+KO}YdH%tIy#T;5nx;lCXZV*aZQe<*yv83(w%1WwOy<%6Zw9= z$!}%^Iew-WW&*$IgA|EC{4dCi8vMS zMHM06q3QsSq8&_{e#+kJX7p%R_yJf|Dld$Uw$o@kt1h0|^NKmYkv_1DZrU9;DKxGq ze_3w_GU>IO;^o)n=90lkkj&isJWBqc47QZ&wI&X>p|8H%3|R7*T&y0QGZY8;88R77 zI~x>jMBI6t2xhg!8J5^|t!?Rs6r25N8n(}5g5w+Pk`;C{1Km$NR{EQ+BvUr}-_{*{ zBrlTHjcQA4T9%~5es=hd0E<%&4PQ41ojmi}fBvVDX;7O&VH}S`_y_E{>#pS&d_v@} zZgM zW`*G@mmU&O#!uxIN&e^2{ywm7!L?AVoI?|y;Rcn|FY#NjW8E-_bVpcOjR|S=y!4QS zrQ=<1zUHy`n3xVzNXbGpJR@Tb&s=CU1S1aq;PymqN+jOBk)g%ihxucE+2o5pC!g9B zj$kp7PVJ-MIJWrIFIA`so_7^z3k6C`9m_ai&SU&T(ydJB1e9K!CimoSzvY@P+bK(G ztxFV-@LnIn&2w)bTX=E1XzF#0XXi9cNq_Y5eEt^hp}Qs0<=gczf$9V~`-gn#Ch5=l zx<3n+m6G#@vB(%r8a4a~;IyDZ2TJ>O?b%DG;m1{om>6fjew;t97$t{P5({MO_4#+` zIrrV4B6sqsS@UA3J(dscNmfe-@vJp?){&2W(af%7G>9mhy&LIR(M{)jBq{lQqINn* zNI0R2?$QVLwAD-kA8%lsyKZ*qK6_z<{CfLzDzXVz5Qa=lx0U4jL%%X}z=@5x^Y%6) zKd&S2O_lAOtVsRZo`Xx;DrMHFce@Lav3x#~jNJLjokq{=S0>I_mi5HNraW`+y$MZU zFPG@T+kwHdMH`wAg!s@l1?q|L2CnyeMK`d>Sz9gSSr|GW>;0k4aw$Nw9_{lZH}RJe zM?QP<7C{v;2{)GD4g4`mp5Q)d6Zd}loL4qb3lwQ4hQgPLpUHQtd5R0cbgeykO0;NB zu%*r(-VV^=?2gI`-eZ7lDp2!smRvWqOrf+uT6~}W`1$T1O9m0*f#YZOBRdP!4DM)n z^7=IuUkh#UT1t z0m|4O83nBrA48Rp(2zW4Y{;P}8?tkR94x72nxOTk6+I@kKxTfJsZ=n&2_XQREcB$b zdekVkl|lh0n%{abrawt#m{Z#QA@qjVdE6}tnAiFt(dzZ*frH|%ui@3(3?p;TYzfVb z-Uc^CP$UU+Q;%xm%6kRzSOPVH;Df_3vq!%BFBbuRa32U(*Zruv(beAD=Oh}=6d=&P z^F+qW!h?M1_PKgh7Nc=Yyyfm7dF>rdY#gC9`kh<<~q{- zZnZWAPK9$5BQrq_;D~6ksCjSc?V$a`Z0Ftq<>CMQkt(jj0r||IKVU*X?5?!m?$xQwypK=|c00K8P?@Z(;Y+>yy+o?$Mwb0Zt=S`<8o+!*d zV1}R9;1k9oR zvO|69w98&X)5|JAQj>O0V7AEu!?cZitgdyNNt zv1Si-B1+1Eh_1>uG**~PB?2It8Tmdo$lA^;sfbXT((;GEpWR3H3Q^Psl&6Vv1GvY13S?tgA;e;{{r z18=Vp|HzaC&hs3nz4L7w`YhB*0C}RzHDjLgGOqsdlnkE&5#^MLmt(d6022EzQ?N1q z+tvU5wzVb$^(o3P3@OiXotCAH6LXfm`&{GnZ1EGk%*EY9SHAah1~V=nE%%v7($P8I zqmHmXPwkl2sc&uXnxwqXJozrOtFT131lrEl%Uy35Vte)>I))b&#IIBxJkgJGeS&}f zd>o&cC|0fi{u{HPAaYlN>Cfv0GH_7|YD13+i^NxpqtR$<=il8P3P$t#nG6%p|0U(C zh!LA=8frK|4?K}EKOPm?(sMyr?J7w-m9d7!bh?!v+lu5J((YA%;rB>60{Bv1e^hCe zS{>6ro|fOz76z7-gl8v^vPtU6%?mW5xdY!sbX60Xxl*0G^3(N1zkA7I{xl-1HsC3> z_*7J~uZe_{aYZ4N$+<^4SzKj1g92XI^_(+zuZZg()`)U?K?Ru9ymk2_uJ9mR9Ra0dUg${3R4Tl8WGwY+>Yl6^doQ72F$+wUEX0UZB2t(0qu3 z+T{Skp?lZ5UGv+_SqUPfp-`RUjjoT~xZy>P?tE|G31?x!$7xnrl897KKjxSTKFl1D z(lOmmD{)mdDyH1s-F-Q5^2Y;$y1ZavRBXUza&Iotzw;?FvsbLAE#y{pLvo@f39re~ zXD)Jsp@l*DOD`L3f>10G&)SfQYRxH$O1-HnU2DPl5}H?gXLsd??e24B;-XSl{JIV#Mg%Caq;WMz@OlRf2xIuMtL&b!pB)0i%PtCaGN#uIM{YR~RSJE%1tgf^F zI*9ub|8*}KPT5c%lmPKNb`xzLH$v2(Voyn5NB{mDv$|0?0TotUZ?%}GCz8qfhsB%d z0*lA8KMr9-+9u*siY{eVzSAb6+)851dYt4aB(&w^%&t(cXnjw|T*^&L^3inay=hxf zZ5s6L^Q4#etWQyH!Qajt}C^%-Covk$*&meZWywdQ**g zV9AUp@!&uh7YZ&;tQ7=xrm8YmZnO*`a%D9>Mw2bCLbsxGjy!{@wqd$%i98>i23=2j zFG)MXY8MhTJ_?oNgkp6*W8^8IMrr&S_k;L9&olu8nGKz1Q1P}IEw=smbCY9ld!*b{ zv7m6@mx9jxm)S%YmjO`%mcpAnBP_jvM#9oJd39YalIZhajooO*Xk z@xo$0r_}v;SxzWhH1Gl@h7I8(UG7RC^;&vQ?noMJP(w{T#mjz|X=->|Lpmwkz4p&Eize=J4ixWvZ}DxNRQ&#$wT=&q zm(^uk^3Iq~5kcr}-}&dCELTTA^|CIPL&3>((AE_1Dl02565paFJii#e@GLDW^ZO%z z0FLs}bD*!b7UPQrrGbfD#Bg$^;!bz6--&4fjk*$YXDh}KR#tV5pBBf_tC`9VJW%xm zZw6~6XEi$x$sSU(tURa<(4+S8+_|SdY^d|+>HZZhyg~UD!*HW=2|~iw*TZjN{OrzK zc#*;;F?Q{Bh zyEENNjg9|LOaALvk02;fW|hSHqoSP_R`6+-QK?^)96~vT*{X-X?y7C}71|o!Ps7#Q z0|9TeN$UqHpSIb_@y7OA3M^hWiz0_sQ;?d@>b$X5V3IYF6e6Ei8taDtsr7%UixW$$ zMHb%>MY&EBz_qsYwo|3-e=TjgW$7sX{F(kySlKezfR&p}_MO63oBUs z?g27H}*Ovn4!t#`wdI#ugcRLFz$l&GQG+pLLvLAO24bX zX<&r-s;caeLyz-GJ>MKjdqyt-8ecLVL3fKFTuOznLa*CG_zpji3ZBZ2d6KErye2#1^NwEv35x+!%AX09%b|FeFVw0pUZls%d?kvJ(0Q~53Fyx;k6QZtL}Y*EL4E~ z(?eut>iY+2B7Hq#a}mq=;%p&siT5F{!(&h-oQ8smGDixm3Gj3)&@H%){;A2CoM5#V zKTAf69X(#5IhTIwifM=xEC?5P(!#4?(p0)NZ%{zUe(sZr858)L+d9xi$L<#;nSrE( zSGU4CqyN;TIo4~p$2&tD_TsM}fM_$!;&RMF3G*iGxR8WNzm*h7mN%>C=AY`+$_U)k zn4&<+cvqC4^bAQ`d%b@r^<*!Vr)G?tNm6|%oc2AQadbR*4!O!FCOw}iUGGCqk+%g@PO5wuvuLQIoyi+bjI2r$p-uXE}XcKbFk43~F z#xw9s*Xc!P(${Wc&o2l$O=Nwy`)H(vW%usQsjGl;ziYgPLuiWiJ|BlsDGP}c2QwBv z>Ty|I8wUP|6TI8<$Wph-`v~6;LF(Wx^D3>!dH=jiWGx6dBI62jTRVzqQ57*!M&7pU zfXiACpoQV&opIYM7UTJnes=vadFQEWvzH>h%Hc0~3IwFH^y8l$KDA}5H`d<7;OI<) zh?$e}uNdMg1c`-WEg!HsIIaWHIyQyFWdE@!1-R1cK{8QT3<$Kn%rPM;onPe=3B<|3 z3zjbl6C4qdyB*=%8>sV1dN0GlN~oliDHMxNN@g=(-ItVmXRqdVYJX9elm187oTS$A zQGIJ2Oh+CpBV|-oQP7R!A1Z_n*5h7PauAt^J*Gfqb;Y2j<*s{eD~%@qv@qL28Ocm2 z={C)ScXOQ#GGVD67+Bk|7mek>V8?ruf#ty{(Ctv@9=pX_um8pUrmu})W0m$*Bg-go z<;gdI%QG}FdHY{35vh+Bmqo-b^fUZ@WZZd1+RZWl3L4(A3l;>e!2Etr6gro>%qnF-uLn#j0#E5DKQ*Y9vc2#`yvg+1^k2zi!;k;~i zqdjP|MI`b&E{j6;PM!(QfhgBdIxueaUsBHWM%T-WtvM9`9Aoz}P>mj4<(Wn8S1E zRSYl(v@LNnrkR4@-@a*r{HPex7Q*1+qo2}vW+NeUDuLsx)GDZv+T$me`t{z~C(Srq z%E3K?q?2DlZ|~50*p^Ta%zR8U2p2o3o%czgI`-BPnxXjkI%j!}Wm;KSzmduG7E!`1 ziAHu%8f5jr=iY+{qTG*e(H@_kivH78A9A}KYQs0~pWS^E4~Z)yihjbsv&9&(V@N=? zv0Do&CoxSk9irRhZajXKGB{_QHeGHVNn6BbG+3YzW}PTho6buc&f$(1n+p z3S9>qq9ygXHdzF7WP`f{hjpnJcslcaYnKzGoVY~6S<@pz)r33$C7s{KFSn1iq-!~i zXA@XKeE*Z30=yze&_tZaGVtmq4A~;%{4}zLk*l+N-OW`nhjd>BXA$Iu^O<{YZ&?lG zMixCOVX(E1+?QQmlmJ%7@1iNAENu6Yw$nA!j|GXI`O3P3a3ws|3~Mhxao~AX;&rt*(8T zLj0zut-b6Q_ZK=c)|kzi1qBLSxx00^l#CKlK4eH2Kins&q}a~I^&G_yJ5FID=W11!xhA96>{pshI5JoR2hMW+ia&!4M537kpz zmHL@5ujV)BSHD$`kQT-<}O(75+~!*3|ciGv?nQD>J1^r}A8l4eYyddam=>$uCxPM%+y z5C4D6y<>EqZM!wv*tQxqb{gAeW81cECq-;Dwrw_UW81dBD{HOy8>`Rz?mhO;oj=JK z_niwT4$L`^<8+oL#R5XwXaNHcrW&{_T+U^kl}RR&hJS~I6iWSrlGSF0S0E5^DgGQ; zDYnsI6x-Pf$(j=9U-`@G;OnrQAEU>*Hbw`Y`D3HdpL=U6oqTWj2#^;y>NK-=gKJUS z`ousk`(Pw&zf@=MaW>6>CTzxGW2^Jm>cy`&@&QQAkIj3rdnGjX(E77rcl`0 z3yz%Rx0{>U$6L?RENyYE5&VZKf2GczYU#-{V&vHjB$BPttf3$nuCx)5RYf79m$5$G zBnlP0JVuUfY8~uul+TBz!KP=KQ-C@8tgax$P_Ia!c1GJ*XsCUc%S>8k`jq}d@4%F@~8NmY7%tC{Q8+YsJp?*G} zycXQz>@~E_MO;f86E53d6FhyeE@-LEHfF4Nf>QYAGa$UX`X|233hV9yBmCq=(N!o9 zU-CQ3!SKS;f!NSBDwTTWVSR=bmsd9p!Hm#>PZk_g`w!q(1a)34re?is%RgkjriNXX zaM2=n_aJ=h-M)+wRTq|X|bjZ+d3(KbgLRMq7+}jo5f5(yhaUpm%K~a2G?C-CmgZTqw{R@CL@w_FEOF2tQh>3RY2M*5XY^AB%q}ibx!j*+^N6P zmYh=nC3EDh~rGDgT$ySJA1)7O>a%cmw|J!(*K7qRWO0DI@1Pdjv> zCLocRl&mw^*&Vsn2o;F9B37Dw*Xl+A|I7rt#31q`l2&?rcBZ(ZA%1W?^x=Ldl(h0~ zvS(AqrWFeDi%{r}%JZ8yPikz^1ZW4DqE%47weMjQeSYLj{J&;M)Z+mts#uC%V7l~3 z)G-Az?v{A}tzWi8h!V}seEC+4M9x@wqk4-QSwSV%pV(pFe(jUF5tz!1Dw`M%=tSjE zS4(^U`PKA#$FDrb8na(u6@f7n@zvfG)M$<TWd#@zfb=Zq_W|x* z_KYP8A>rD6`ajd7`4InPwVVc-xGkh|v4^SM5IDJyw&#S8gBh^!7|K3PfPDv;mNJK5 z?Ax1HvJaEtttb^A2?IUyf_6Y@1UF;6|XVZy%9 zIB1O+^kMGwa1Rh(OlTuk(wm(wR#j8ia#3U5-|ZdD+h=2U`_ebLiG2!nvGb^kuRZ+W zoOKddosXMv1(mZROR7>aw5zzD1q7+jwJ7gl)&GjdkHxba@*yccJ|LQeDWy5@1)@`f z!yiS*Mo~m~VD!cR(pIJew#v-PQq2IJPF4_uNY(o3VR#5&I~Pd%M$ZkYc;jPx-MkSg zn0meLM|}-L?=I+;5@Kw0bald@_#_5al`xW%d~n;Hrq>U_uYI2^@2_KPsuydxZi&vf zDT?oJ6sU3owH@ZJCEyPhFFj=QTQO=nmJ!VHQCw#9nb4<+yt0k)sO%Mi5EE->BkC`M z<=+ky&W&~b@Pp%Jl*@sS5rVw^A`lc;-i_&xBah)3F4)`$Ch=lVG1*T2w38PIfQ0h} z*Uo<-ECZnQU7mY)%PNKCYDOpZ_;fi;8|l7_u+GFiwQwC4$c@@ZoqY72_D=w1v_D8S z@6ZO1CSA+OwexBDD0=_(65oMF4T_^B{CfZ#gr?$pbwTm{i3H4M&$7ebH8lM3^rb%W z&Y0|wZZlRz&T6wrf{41h0=MI4U294F{T{>L(68V*P zxTuopxttl7fi7^sRdh6_x8DPjfrEotb&-;Y7@R#Ext{xHVjy@fA=aO z^E7j_te#q0(l6zH2ng#VU|EX&kt6*bvr!K-;5DO7Q`zc3Sa1{WD8AN`zpX`_4$VC%ndUYXka}7;V@5QVs`(l>F-P=t> zhZov5p#vU+HpZ@Zhe^F^W8C7Xr2WDNQ#D+lm}gQH5tAMV{ezDjPEDW6c#FJi58Wsf z@0td*&38R5wVB+f%Ch3Y&wBYrFRzAuoHLuLU`re2m4eqUy(mnjo!%jhqHaSyl;=pG zcOBn?2`GL|(aeH1${~oa#nQoH9#UURcwd-te!3={{x~&<6lS-U>{kS#_tXFJJ%Q_t zTM+sy%J1R{2=zoO`3jp*ZA}sBnl9$^Lw7W9Js=xq1%Ny`onr5O zzO|UkJPpI#T}0^4+)55L%nwi$hRQFaDDZ>-SZwy(7>LRwlbP%9rCT^u&1sJG{R(AA`%m0N&{JN9*sKoSv)jnAGn7e^767_xJQ*HC%FAh9-#dda>jSS?Z&YpyItsbA2 zg?u!>d}=K4rv|_F2OlL8ye>-dJe;UNSAN4>fJ!-|ne6(bdVp4UUJvmLG+Z#!iD&I~jzR@}I}_H5tu z_u5o++O`{E%nsCEf#zn8O@5c*NqotGsH`yaGnV_?t$3!916h|p*P&{t?{s&b*Zt-I zmBIdOg-N&2#kg_4UlWE&m`iw@CF3F;E<((#D1o{)#9tfYztKWfjCYT<3UNRW5elnX z&~VTLJ3j6TKKcmnB4LRj@SK0dsGTy(mi-&;i72kB`HmV zvzbs&07gLMTSEbL}pMon4tX%x`)+tTL6wPRooZM z=lf0%ZsyhR=~YMYkG(x$Gbf&X4`chbiuWll9v&5RIK5nkg^_>zFo$f{x59==tmKvH=0)~awW!|% zyseNM>G*(E_;G@WPnR%?jyjR7@cyBiZWkRJ z*Q~}$(K{8hv8d;F|uaQPqw`5Es_sW?TsV^f(4j zq-v682!yi~iRdO{ZT;g;9KF0QWES6)EWHvH&1|^-#&5l=^Hcc+K!RkQie#1CYLl3XZ&*^5bV@ z5Uo@r^VQldv43aH_D6u0AdIA@rpC3}JU)i!=kFZY`)ADa{u*64fG6WY9j750`%DA@ z$N#JwY;eKXBp7}4@yyWwG-_otI6rx`$%oOigGsN$X`>B-@;;1}L|LPnldv+%12dl+ zldqi^Rb`?gkgpvX#qhpx$RlW5U$k4+cn0H>0^oJE^!Q`mK`L)^A|S?yY+`K0tS36y zxfy!cKwj?kLGgL9f1ppt#2AX9%0~Sc@H+ed1$fmBs43xUtNRj0c!zFa-1xa$C_P6g z^Ss`C$->Zh8fGEs{7}_krz7rpn&W><>>fC~Kh;GDpj2LtAW^3HqEp zmn|eJeT#tHOU>m6)%QTn=r)`PNc01fk*@%b*0DfI{>8s7E3T(i9Nn-25Aw~xa25F- zgvX{mC;B^nc{g(`vH&vmJN?*}4giLKt?gIY+A)dUUH8iZ*m`&%=%>yO#msY7eF-RM zFOLRl9e4mbHgH49e4-Xc#N_MfggbwXE1J)OorvuYZ?)p_ei|sMIkEnJ0<(>d=P=PK z$x%=`0H_F>EM=f`19iqK%ysHf3NxQSLZr!&=o?dr=yAjIwdan~$KibukbU=vf6}bs z{%HWzl>j7})dfS_RacTu_eC}A<|&G)qA?4f7X0Ua{Cy8sa;PAU@bXp1FGzWmy?c&CCNB(zkG!+0xb0QRO z8#GmOKq|{e`T(*ldHznOYTZRboAu?4u&=K_^WDtlV#2GEMJWQXBI1wDYLj9>dp3JQ zfKs!E4Xb(@P@NxynHBu6UI0=un!E(WN-bcBh$YticqUq)!wanHZ7dKG8mvvWhLpR} zuwq4>Q42q4`4dCFzuAf$1UQRmbAFhbgAcMQYFyZH5QZ(1^mn5zDrR56{L4E7yQ%6u znzA~iZaTi^;eBo8LQ6wTG6dj7ZGNr~w%qb09>WZiVo*f4x7E*Wzfr}MKz5?xY2d;5(nW+I z$>wE@9OyH~e;e6AL44A_rrp0k~5ZU>4PT2$0EX^()2je|>Jq>nTev$i%Puu2hgzw3wEOa-msZ6#LHEbJ3 z>h0>+6nai=_u0K!@-sy2Pu0y~`QPzh^9BLt{GEgd;2R&K-I%4t166tDS~I z-)48DMgXN%_Hsw9^wTfVA|3a0c zN!5_KS@!j=1IOjt-4y)|{<#4|^I%A-INzYiEe+`QvPNu3%W&)@zM}Gyp_j#ku7Ez+ z=cg#=d_eaC1lgnbn9%$TfU#IHYmKirWB@|D@Kzhzp2}l8@lEGn@{_)pL zR1pHk_xAVPZo~KPz-C84!{V;Vy822C$1Rr_ftDhejHe6t(%7olMgidqW;k^;R{G(G z?v6G^U`SrEO4CKX5+Kc!*Iq$H^LipV=wMK6;YWB4S|Z&IML9npD7eh?AFG83SfQXf zUygLnkJuOn0PqkTVX*bB9Hv%FDmxpY)4Q=JYdx12Naj6w`C(A}$)|D!7pg#&Gn!!rd zLrSw*7aR^|Gd3oKfeZi=H=CK7oc;Nk_3lwZ(&T%1rwOYYV+6a-W2G!Hk4h9kEIQ(_ zI)??sIzW)dV|z@5K{s|zja{weUiK)qEBVP|jri+YEk1z`Iy1G>+Dd@bzq6`ZaW%Yu ztqcEVz~+tqsLeV7R*k$KduhSDaIPJovBjUc-IN*W-6(~o#M3~S(7on&eENQj8>Omb zepRhtXC2yFnD99T&yU}D)lf9#C=x*TX@lV0l(HMn?&U>8;VEeSn%dt&t*n35{iUr% z6G|rlByKbY5U&MHws57+;WDjFS3`A*P0cEuAE1$*w&5Ul22}6<=r57j+NOW z{+fSAX<)21tX=C-f*@KCPk*wz-VHmpb+X|-eWPabQ%0u$y+?GG5n~T`C3-z0yusg& zRWm}Js+(P2L1kKe=jRoGA+N{&Kn9R97T_SFY}x=I3&k#%7h_AHmwhh;xGV_kcg z?B9182kTAA*Y^st8lDTL!IdqigzVrs#p{{&0h&Ih+*Ty*4AB;ofiGH0QJaOcJ@!Nd z^ukMua=z? zqpC?qVU@(HD|MWAOz@p_WbcAXQaOwV8kO^KhmcrKQKGxT^i8*mt4!VAuYVXlSrJ9V z&WDW92Jz^Jl<~Su#;09oqr+E=g%^7XK+o@J$N3TAVNNmdGcKJQ8JB@w1-lBRflA?v zQI`|m?S{E24!jszZqQ0!-x%&yAv+bb{-hfW^f+a2#7ANYJj{qkMx-Tc9xKrRHe~an zN&s=T*~%oqC)gnB0Z8e;-T1L+m)=q6-~gg&l|BQ(YM?8#x;aQ@D*K9KKD@Nwc=yI1 zsnHm>ZW9xnp|?~2U-tob6@G zYI-~>Dr11ab_3lNkO@+B<P3N8Uc_UxZp@d-o&w#9(}Ep5&Fofn zAOMs2`ZdbvK&6R!I@p5jfD#oA>j~-P!vxfzk%kcJt=TS##B&V}KE}@>G1Ek~O=Dja zo<0SM-eBI^@tRJ-9ogYej>D3PuFk$P5e$i4%fpdCO!9$Fj5ORA z1z%b?5Tn-Y*H{#H-(R}>M~95eY*NoP2h&X%Uq9kc(DWQ`NlGwK^{($TF7rEn(C*`K zoc?0NXt4bPwWDwFnvA^6NQDlxvrk2fs}Snm1;4+JL`PeT^G+8B z+5|D&Z#a7Q1#ADx=Y0M87Mp%oozaa-+XMpaoG(SLEeZW_&j{#Ju(KwBKT z*XM3LzE6!bvAK5|x)gdn5Izr`R^p{ZlZWt7e46&LtU#$ev?ydDvp z2H&HI6MiDW1hWJg`*!2JcS!pU8fl+;Ih(mFxm3~g!_N6)wG}Q*y_Zc zBq{ViWVfz>9f7-_A~j>vDhtjE*Q?)nx;PKI-Ri{u*1n{EZfEJnaOZX}#y8 z&_b*MLu2`?Fd-@TOKuNJ1{>uMhIWKByD}DbUNL4dl#`)=TU5zid`7fji*XZ>Ozm#Vshd&;Hzp66*jCavm6g`W z%aCLJFziqGJ)fBx5gXv^ZrP)7-_i;QuSW}GNek!w9SrwVo+`tIiaOz>TnkBVGR7$R zQzMEXgpqwt7&)FSmO6UX!DgUY!WD9gG~=|?>_iH>!->L(^3$Ee((>S5&kkL8C#s2T zgE&0zunaFRl<#k%1V?2_H-1xSvf5fhL(l>Oq;Awm+MvIjszMMDgtK1fx-0>owsb_& zyO?gd4R)FR@SC z8HW&0eUpiTJbVNUpq*}>$Q1DG03r2q-`b!tLM9IkDfuq+sr|VY8vD!k#X(*r9;8jH zfR;Jz5s<=ieS=B+=IrjX2l6Kk7uh0GPzW^J#bE$VtzX}q{c6>TT$lin?h$-PoEe3u z8ZWmJ2NK9ZsTGYrQ|iAbVe^H%J zaCW)>s6(&2+9FTyBQ%J;Obg&Zl=-E4JQ3U0ypV%l8`U06&`9>8Iw2nmU)v!eIx|Ip z7%jZRmc!dqTqDPnc3KzWdK0O^ndek$A}}cm4z$$JVf#DEswToKS!{2&&V&r(sy3mI zD^4tb6DkG_=erO9!2SjC#w0D$j)rb#-ZA`@mtV7g+ecZf_+3Bi;mygfG0o*9?|Q-9 zG|kShgyj?cBXkepa=-dG90rBVWjW>C;JOwS4eZ^f^evOrwPy;gj?_kBy<})yXZSOMi=Xf9jb((&G&QbSL|BQ)u$^=3(IiY941W@|A+C^5W}c6}JEx)isq- zu0Vrn5eGU=XmdcHir4uHp&~{q)+b8BFa-sLYKgqQ75&a->SJ96hFltHaFE5Wlx%zV zEi(!sVrd7p9VevN{Op1N(iH9ZP=b^`rpHqD-rn=tOJM!4dSPVeL9+=U%+LvCfJzep9JW*C zTn-1d1WLrRcUMi8{sz-w6}r9x#+|mfIJf2<%F**3>nrL*!~T*Uf@Y98&qD<5yCTfg z>4p_;TUh}Ubwyorelzv^{=gXaxz5&oXPU`Qp(D8V+(Kmk2HcfRnV zS7(l~JarHKS=78}-Uh1q>aZ>b0kgFf9K@jv`6rAYo3Dg^L6Xa&PA+Q|O>JJ{JkP$= zK%tqy9X8!*2M;Rm)+qPvSKUV(kE7TgN%)N!iwpF7q@&3mTqh^kB6+~`JU)J>`?pH4 zqxW*Z*wRgDABTb_smDFzXcN`#_fjIH@2 z$Mff1-b;H4cEtUNa&Fmq_bTx24tn>i+t~P1?fAshQPlirH&R%(43Wus_r7Ho)XWT_ zS516ErR)Vz3`kO6DT#h&ri7*-Y)FgYI;MT(^(gLDiN4o-sq2&V2=1Ta*{#V}z3Dsm zn|2Cqq%e!F0p@-a;Pt8w_ta=(Xr@1l3iK4m4v37?bzBP%&WkpTyTMK=C%ZDyP!}NJYr>jny~aRpM=y{JefrAn zQ-<%jay>%jfqEgR)8_D02kngEMxR`Z-(tZDBp*F?`eoipF=~zWPLUgI=nD0~&Q@__ zz!U$cbP4mlP*6~Y731dG%wp*I6k5!foVck2_sR2~)I`e5!#uJ0;*cjIQ%x}j>GorP zY#r@=Q%kP2Fwg8MeE>ky^R+d8UJT2E!6Xt2-=6dPF<3e$_CyL`JWMFINUSdWOVcBu z;CzKPo*gKE)QO8HunH(x&3Z(oc;cVanu=g2C~L zn0)E*E|Wrw^{PX%+);IOL$C6AFT-iqT~k-2prCMfPj*XEFe3gBz_&sU@U9rHC1k1B z0PS1!Y)TlqlMg!XGXMMYho8ST9eVvg?&EKpMd-|Ha^jG<>ob|SfFu`$Bvp;E{NZ7+4=9cX17t7(`>2%)I>ye;M9VeQh$3!sbeE zclwE=Qp!P3uJ31zVOb-E?6rg)EPK`qGnlq9sPgX#9(q-#zhwG18cN!}gT7Dsg@t(~ zHL%3Q#NjbEVVS%wznl~`?apWyw1HRL9VvF_zR(>&JIe~)D>=bd{vM$usF{w769W+0 z?5c#q?jiogOrFj}^OrXF)_c*r3)k-iv&6@jeEarq zpE#O}Kiez1ge?hd;ffG^g_S-({>e*OQX%*@q`<@ETcus=sT71y3&49!HVPjq($E(< z=ER*!bene2QT5cJ4#q>|; zj~7Qxp#Ok)e%DK;L~hh~ZUB!}MzU5TghN|5D*rEqE<3<}5i z*9}z(3hN*WSK+7D!Cuxsd)7?PfNB$Sa&Bhh?i#*%0muROe4N$XMiNl)O3MNs1cekhs}mI+4R5{D zK+w^_*R*x3HB~JP6$*#hU}xd*U)9__C#Ban?ry_^O_eJ#6)K|hKBin2Os49 zi#>Y*YHKdbBUz3Vz2N?%2EkN?Mxnj1Lm_$E)<&9^LCaZRd4BjYM+~)Lcm&kr$IShy`d8>WRFv(Uem`;~@$Z#MZF4}~wl*EPog7wo&kxd| zDt&Fr+@LhSLczr+t#?RQCREee&jRr|w8#n6mBtMtsuAp8Y*nJ=|6`dzRasYNyVu3S zo?gV2J-K$fJ|E=m83PnF8Ck)->y2gE^CRA9G7~|RZ3{|lY@5V%%_AL6Ot;XlXXR!W z4Hbe8&jYun=NiHjcoPmbMtG1pg#Kj^o&ItOKkb^zk;E(uWyzS2cdtZ3U`&~5Im-kl z#USoyw-x)pyU}o&5^C!J31GtGt*NnZHQwq<&pX!!9((Bh^TR7YL@KepCVbHKZAGsH zJ+qtT>$GwJKW!TAB+ z?`T{{71;UrQs<`wtA@WzqpA=B7&@L|*D4+n^Tgx&V%W8j+!{ZVvi_NyBcrG-I^ zhnWFQ@2Wj$U%m`PrE;)H`H=M+yw)5&3cDurS9sNFzU9zp6Yh1)%^1J}lX{!3VKzG% zJ)sq$Z9$w)?QHJQJ!PY~^=i-7^I~3~EJ7z}S@(@e%Ms^w`CAQy>5uR02Sc&E#qCc$ zbqEcgD!cbVNj#Z4l}cmkP!FV2dw`5@xk+oCL?0Dh79bV&l65gdu5{( z7W+KW36+(Yt91b zj72;q!+H8U^EYyvksp-$c0e?JTt2v<`&aIhO$RYOJzed*;OHPO^l^70EodrlyxqL9 zs0h2VtpU9C?zo?Zq22#LiBa`53Cp=DJEE+ z8VUTB6~#b-0uK-E*a(Vs+lK=2pjtLOl;>Uq`+S21G~h62KQgXOJrzBAt93F%&ElrHW4Xm)h*6nq2TNo?N6112FX! zZS`V`1cM|0?9%}tTgv+9U9ZU;w$U2(za~u#4+ei_{~GOkZgyg{J#OAXM@L^THp>(h z>Y<{2Ip-A-6X}|g6H{h_tNX!jSapuHKD`2a#-QWdUElI*8?2Y<859zXjb!Zhr)#Fy zT@Z}515ise-}nxptGTB2<+to<;rGvM2O(k^N(c=<=VF2UHx#;rWExOg6rhVF zWi5L9^=X?;Nm4Z!R27{Ocj6hr$pKS1LWhX9Ve?dj1!7TI;PFLION(Z%yT9P!>blNi zr`ZWsUBUkh0?x38QWTbjCAn`ch|6PI$?V`W3Kr~q@rO0I#IyT=mq-2MeWx%+%R}AB zKAkPX$l)|3Le5A|wef*#3#Ew#Wn7Yj4e^gbED=|8G#&m)g%lKaeE(yL0z0pXuXBI# zw|P(5*u3;#Whph@Xg1ou{8@f0rx|Y7kX(;(TNSs^;H+r({FqBZT2_aJDGjoO9xqqa zaleSim?G13HJi46v}KfrVNy=1PX=;uy41M#28|m7iL~a1Re4_X^>b9cPkmH>ox(?>?^$ z@*U;n0jop(T?p-IgKmmHG9ShrboO zPHtImhJa5O=x@|A?8M8qoQF#FoUh>exB)z2Dn~2>1ZD=g(ZBwc>98PGVV!v?_U5YH&UMD7WkY zD3IMW2RN5^-c!x5hIJR{?!mY=uBnBIj=6bM+1by4$3{X=k7+azjp~;AwAs}y94{1x zNU*py?8gghXJ==n60nm>1t1CO zT80{HviI_FA%^PM^7ndA*6(SJah^=p<#L6+sf2z#m1(|cwV9xDxD*a^9$B67TK}yx(O`S`ns|U1P#-L#i@Giaconw1y;vI&;C3<2;H6XlQ;V)1 zzy^RElLG`@D@)NpY>{>l=|ufSO)boji9MPAGa4snGb1)acx$)_4O?R| z8smXW!wJ@4_rJHT@y4nFp?mKtAtYd1@h_~Y4-)vNc>#*5|0?h=dmr}H7TTrRcQu12 zQnl~wd&3_agTzX7;NheWsAhj_0?$%L_RH}HIe?E^@D4mK)&O_;|Jy?}ST7OF`?8n| z@ZUc?YG#XI3WDrFOMLUQS#E&0 z{`_aS`XAq-1;jQv5$P^2H;lLl_KwqC{rwMtXlpaGv$+ioh`2G|)F&WZTwDyfjnM8u z6+6(Sh)TFynsSYgm3ruLcgN zgH@&Oge+zWBT1`Z-@2z;NK+K2Ji>Io({l`^v|E z#mCt5sUxbjwfVxwp2@gFEPx!a8|I*2+V4XWGctmN!w`vSY2kTW+>-d?U(bF{L(TT( ze@f?a!BtdD*k3Kjb98jHKcC!q)42(p5Ed4m$`SUue|i``lF(!MH8PU$^=n}8?}(wH zA%B)2AA1Q#lHOh>>v;umD2T|%Ce3TC+*SSfUigFF$m{M5KJ<38UDeG9s|V3&1z@IU&B+9-VgRc*AN364E>UwSNUB^kMfVa2b6rv zl7z?kNbb{qKxl~`P*OsjpZ}ADft}F7q59HKc41ueDE;A!ANgof58{DLWu7ytH2&|s zCoL~FY#_mpU;T`@>Gws#d>o6wsy^@|~r$mi^Mv7E**_2H?lt_)s0?qk@s8vy7h(0LLL zt%dcweRy~RYSM?+@cQn|%p4G9xsnFpMClhpycNj+1D86PFwGD#rq#I7yruSn`J zq2tyyH+N^JCDp0S_p~WZ`{K??E}QP(dfJ4b)o3#q#n9r=H5GI(RF;8!pETE)P)PAlV6Bn_k2>?m7lv9tEx?#}Swb!PZ<;o(aRe;?1g9($B$X}@0& zA-Q8MN{Za8t#C>tR@qBujsKkSh2zc8Js~*kGn}`~8i}hAv z(hMol2Aj2qAr*+3H3nML4&Q z-stDHYmfE!_rEHi&CinD`<#EqtXH8}Vq$W54UL4Phw19_0DHdkZH@0UYq5TX z@f|2$pW7Bgr)Gq)gG2dscQwsZpVFgS`CqKkV-pI%=vJB1#tch8O%6J*{>X171#U%Ee?`#^H%D@qXLYj4FaH3z$8K@LtPs|fgNoqbRT6`8l zH7S0byPMwP9Z$zGGnMf>OIceHH`^WMoZ#H^%G~LwTyuG;pvdwhAFUt7J>@=Ib`mNF zLxd)iKE+Ti*CimAOU2j4zRAsMd%SL$#VH)i+yXFuQKeP z=wKA&5nV5wT9IfzQy1PcGxmdu(9n zQy}%^b+h{X00%mE){AI`T+w-ZsL}hj>P*c;cmA1NlOOnW=sF&ajm!JtfQ7X zf5aD|T(44hnPRq*BtGH&s^_{A-!+=-4Jg$rwC4w3>GU!`a&*>)+xak@^Bl9=0 z!8c~lT&HA(I%zguTM0fJUAIDiB0+|M`)vJI>b9SgSfoHP)%)DxV>7N;c}kp#k#P{T z`5Ld*0~&R(>QmXUWun-06uSi4<7nEWwX8J_+&Mtt6YXdkJ3pgekxH#;=%77RfGX(y z&C$%*>}x|Fa28@nfu0}1X+8J|16^&pQB6rd4W@?S7f;WY&bsf@NoC+iK~>V zUa(%7WsNy#@lzjp>Q2!q*ED5?LgzCB&6p*D>_OV)iCTgVjwm236pad{JOyS zE<_CI>;#+4MO5JVwTRA4ei?tXO6f&7oP$epIYb}%PGv(;BJ^_+^Zewi&hv42LZ(jJ zS;vlWC5>k=7-A8M4YgLIRoSRD(Bz(;T;|ERMT3Fv=Gzt-!J?%45T+!rI`~(LDxmW= z?YqbgNG@=(b97HG+gA+1kr*_giDv9bP=AU}SNqW@jg}$P2qCOn)dA;qPoUOXv-R;>h|cKoHzT16Q3P(b%4_w&Rt*5j(|+gr)h8Xm7biR zf72q1;*zeMl_mg!T*%J=r}VR_x24mtsA-b&r*u0N_wZ3%l6ZJv64jf|qb7UC3zfr7 z*oI_GWc##97pmWv!tTw{To?}9T6gZPu+|)1gQfgz)XQ1BR*z7PUiinPcwi}cTP?ip z%|Ecy?0tw0I*wH5mQufe*!!{3&%_Youtjn2)t2wW%}oy1LN*Ph^fS0H(|7jy=nb~L z@)#$U7wxxo*=!2tb(utm;8=$}Lcd>`4 z8LJTBf$rTf#^khB{7gs%k+*6xa}%QXuk z?_M;k)}y*y0>^llZ!%BuWhP*c`_n~gjF;=Y{=LMlPCRIWT#NqG&Hc@5)T zol7kCG0Ey^sI+75#_@Zgi*Y5RH3pS+qmU&+#YFxv&fW-m{No!z=R&GqhYm$Y-BCaJ z68l~KYJB}MM|LU^7ouTE2ow}zRkM#LQETeMG@t7-vh?~zA5jhN+WfNhdJvKZ{N@s) z@p+dGDrGpYwqNgr`eX}tz_$Di- zhtE6k&X)7f^y62Bslog#nL|Z}*xR2q^;R>N9U}PR z&v^$NAnG-{MW<1Hf%kWH)1GAO36U7siF)-i+2N3ZPiUsHLsrY4S1}t^kT<=szh+x+ z8n}BZX6MtbXgU_B)LU;G8M~$OiZ#ur^Am2g4{={48%g5h1%Op*y8bB^p2bTFRWkBK zb97=5+y)kd-Z*6{EEIEW;*y$~ZRC}Ecc(fh7{>YsjT-Z>QMVWD`8ta{hzQu$rpZB` zb&X}QP3Pyd?tx9-*4V0;pA=cfy#2372J)eYE^}-9pabEBS@sEOoO_j>6|7H{Cc0C? z{w~KfTgzhdJwiM>FPRtt>9BC4L1J3F*T)M^Jjo+%Fp4edD(S3T)dn2>kNr$<5y2q_ zi6RbPz75a|O?G|rKZ4YHJGLBDLo1!$xG)|zWozOqZ}guAh0_+@O4x|=`Z~CRnf(>x zi->3=Cvm)z0f<&EmaaR~Xnj4{CKeOut;(&*?cIPYtv$X-PfuZDun6NdSlZi_dMbmH zv>^(c>*z)^qA?rt2fMZ%)n9+;(LQ=-`-NJK_}uX$lgsB@>7 zMH}h1yt^GbI$1S#IrvZKjoLPr;7cj7O-RVdhUQUEY;cxR5XQOJ(pKQ2plcM2pn8nr zGTX~juI@H$uaL;M>fgY?c%5#irMcWiRj%DJ$ynOk4-%JnoSwU(AdS_nMdD`4p#^|# zP}SXcIm8xE^lC9CR}bUifVA6oGs~?@=M>1Q#J3b!5)PsCQwAYUqqf(%R9ir#5UVGk z-b$l>tmXW$>0(v8d5i4C#=+}Q^(oQ)QTs{#6H&!k^WMjf4`zO=h8P@ck8$vgGV=%q z=h;s>Lxm$@*6pdLg6^3=EYOBxASZf-th-*oUfkOs29EOh4^+WXQ6)an#tTMf^c~r? z;n25K#h5AADv;;Jk~e-zbUvJ)DlvK8auvN^^;~4$Mjoyj$af%rqUiqZu%%aUOGYMX^C7LAQ_n2Mp^X;oQmgMMq%H$jY2 zyw*672OM-jGMkZt93C9f?6l{R$OdZn%&4<4zR@p(zqGWA39?HlF41Mn`NKnuA-+T)YmTxq!Qt8P8I{t6}<0Ov=u(ubW>8h(A#<$n48Zf8TK=u zX$|rbs!*MOtn4`ToOtZGy4511?envESML=}^r`J2NIMRW z@-gFDHUpTIzvpaPPBIxUZA0YlosEwGD9MlH!en)P7&vf!8r3pnTvlsrMBZzCxCTu4 zTGI){Es8eNnH-aLu5ZCGx-?T=h|ybR=2JTDxm>qQ_*RX}zJ-Knyw)!P<+@Y3qDF+d zS^uZK?+S-2;MxsFj~+yHf`mcTL?>DVQKF3AOY{=G4kjU@gy=$o6wzjkUPtdW!7y47 zC3XaJ`M!O|1P3d~P~z+ial}}E z$4p3M7h^L~qs4rez%(r@ap&-(z$J)vOI41ASe@h^HY1+#<%bToYI#^m`_YuevTMp9 zZO(=2kpj&rJKrBi6W;sdXy>Q7cXPiDeXglDiK~*@oxMHpuu7!?FuO3xe8!3g*-uJr zIn!SQ-pQ&77_y?*fnm;Ilg<2g#&c(ON(p8%-enMf)}F%t%C`@hg4x3GO5Ixz1x+fy z!+ML^!?y5n@k7!HiUP}P`4N;ECw1dr*>Gehb0HbzRMTgmCf;cbw^d-9m=(>Pb@FHj z)ONruy`$NNc$;ZxG9q$+s;bnKfv0Jts;b{+F%o;Rk5j+Y0nt(e=6g?S+J~jdy#SJ- z+rxmjWw(_IvStst=+ph=w&B$lv<92u<*0-2b*2iT-ubO={IDMwwnHc_P<=fJ@ti?;)b-=edPs|}uTsb%g^`xIRL zm6YI#c?Vml^P|8iupAFq<+fkWpYkDaf!`6R!kQD@RS!eIlDlAz-oGQf%3^MI$KRpj zPK0lz}(3N;s8Y2EG5{nQxi*sBL+AswWeU@*~!~UDG9> zx3&K5TcN`$wRuKXFTR{CTPLxN{lRZ05Md~d9k zsOgwxkI#&IwD;_me+#{{aAQ7Rx0BCkcibo-A3m{5a2tF`on zC@lCcT&tyb^_(C|V3s%})2>OIP|R$JShS{No_t{?_<xgdu=TV&m$2f=^)&wN z-K=o1T3~)|4=^bpyNA;f>!fS0*%^oc(UOwehR_fG< zE9_Fb?C)g_SD(DW6ZcXXL!UO zoal&|^~doZdxQ6z82oc5Ysv1HW&=?|2FN%zLlv%C*X9*IBv3P`-qscWVzp3qQ+UU# z7^843#%!6GgeHS}*MvK^mU~Xfz7yWQIc{9k2uGP;&9sCy@!4!)L&j6YE!luk=#*RM z-VFJ&H?Zh+C;nV4RUm4L;`Ztr(D+`=vT{6fgwoJW5Or>%4AM`FjDg*5nNINkuwomA zGzobhn^LSg3Ax~)7G0`e>-aX3=ml#v9uZh#Uox@B&@qs3_~=D!ZpZ!55?J6NkSb|g z-uD&qkfh0l!}Uzuej1NKYPK3tMff^%Xbm6E0e{6YLC5fT1i7*ySY_q80I(VN57QT* z1=$uHoUI)Ykh_b1n!0M4uT+ijX5R!hH<>8AABj$H)OU_BA_r^nBIK(?kJQj0^AkvQ!W9 z>FYn;Y20llHNOhknOVJ{ZBF!1<={#N=z*tJuN39#jJG0zQw$;gI{`u0V4&ulZjTGpdD}tmq!+jd7#_%t&d51pY*uq7tE)ZhQ*VDrKR~-}$25az%QbXNia~Yb zvnp(%?QVDEoUKu=R-;8kl3s89p3izP#<*jAwBz6(RxKw3L|s}2+h^-*TmMU-v{M|OD#PXnuMP4t6jk4XT!vDBsc@M+1N2fw~>i^;;m zS_Ajzn)OpQRXz)4U}mla7)g6~61P8e)aNW)C((w~eYA9-LsK)fK&tM@imrUPPEYf# zR6k1Ig@oWXSno1<0xfEc<<$aabU`qy$qU=CFSi1P>!gX<)_289);2x4l4JigrnDcc z;$ht{thF`$FnKjQ7n75O7DBroo1=?s=ciJXq@IxIiQ3I;6kSwjLPu;d54Kmrum&L z@q+OA-~sJ<^c`F==;SW4edJ8sRsZ`@MoR)!$zkwdgsuMZ5xhpvO zWYo9y6@Bdt(dFqko%ExtSH^@@Rc=Hv@`(}nDfTiImDl(`;Oz=s=)q9!hs0?-7Du{!ov(o1 zUqO>@Ro*$Z)An2H2fpu|mU?RCm6n_J>-Ro~5e+MY4^5g}W3PeGpL=l?)+S%jeyUT| z4*qVVfd||eoxT=V2k5v0@R!FXU(6B-qU6k5_k@31eR$Yd_g;14T6=pNF?-bnSsX9y zzGcLmq-6_Alna?1iV_+)x z{H*fmfPN9kg5Mi1(C^~KV*cElm@X%xs=JEV>4B&E$jA0mhz#Z98w>gfxGLo4mf`4_ zUERmx1CCAhuIi-%@{s@x9NpT3)8Zi2%jLL7PtEd@!fi-#&!64w0PZnH4b27ooG>No zPd{oXi?a8*Sj4ml-0MgCDE4cK@)(!uT$DJHP0H=_SR94LHEadkE5Ab6)@qRsQ?Xkc z9wnQ{W(@h&MK#G^5@F&l;Xx={&uuiijJOR;&TPW$G{WZ7y|gA;0a<|mKe7N4M53!C zTwjH}@5>?@m`qG;7rpPcit)^FKk_V8qJ_4ewsl2>(38vd@U`g3((@aXQa7OSk-^n2 z`#HZCo{a`rv@%~@T7((08#}kcyCpyu1&CfzT4(ES(1&6P_g6%CYT2nY*I%=Kg~WdR z0Pu(t5Xrq14buwCIWPU?Zpn>>CFG;zG!BZMwffbQN6!=--Wngb^p_=*z?ViWCmFgz zvfzrajzXjM*0YfzQW#92A;5QJ0v(WB3RmRVup7|lLTPfOYcNtQkC$?~U(_f1a)qba zBWc&G&)#Od7tInduS<^SYSO7}C@vkP#v^HRq;Ou482fdzuw2~cB2*|fo?B-%I=3HsyN*`x={dsI7Ms}{j~N*FTo`*Wk7Igu`vfDE*C z*Rf5BTc+7?Kl@lyfxlP{}G zRXnZ;6Z*t@bdi&dgeN+iL-%>ODx}a!{Y|@S)A?)1MNr^Q!;(61PLb_#TmZX5U!s1)TadBnI&LrAA ztW#z5l1{GQzvw-{)!CI+JTxRF!<;1^t!>lwCc+g%6D0ij=(4j?pEowe^U!VE3EPr? zkLccQtvCr|pLzq7h90%A;HX43mn$k<@?sv5%!pY~n`iHSZ7Do(Uap zOngH^Y&8c>+_cZUz7o=?nH7F-wJ`bEDH(oS06aQ@gXtAEhrWW%O#?l|Mcb|8)8E}X zfpW5Tj1jWA`&2(|*%ZFpapuK_GeRRWu7%O$)h;t(Vh_MxkBtg zxG`JXv;g{E@gt!m3q{%2SB`OG@1>(%U^-4iqIy-SPOj<^-`sJ$D0wlL!^{ZL3$skL zMH>`+u|%HV^mI%S{?e82XFpKLP!ds{d*)Ln5~xU*hk*3y48cI7@4fvQ6H;^~_zAM6 zmouv5nGE{Rh9ka97e=kMt)_v(QIwN$&vj?YWIvvxi#IRYP)GiU**;E}lyPC|; zAt@T51k@UL2fG;1d~JWmYu;rHUQvU*j~MoyT#F~XmqzDc`;~tZ)0Bz=PnQnm%lvC*WxOSQ|na<`1PKoM8zQ);Q?@ zrjsHi=!x9NXLlT$(FM>FJX;YB?c08(mY4D5U&fRF8w5PS0nk|kPgfm`A3PLgl5~RDn!c4Yp zifVwv$WFS1L5G--^z*H)sE1Zany>bGDfY~QwQle4yNzi+RlapA57XfA;8!oVc`Qd{ z9KHS%7~!xc_&O0)5L&ZV{UAD z08+P{@?=1i_OJ5Lhu3lh`AzCx&|L3oj}-|!*QyP%Y5f%*G^IOn@xufXusQ#YvFEb9 zvAg^;7G3RcciPukN;-+T79w2Ub|~{;FuO)OM>0%PcakmRsSR|HyL|ujt8I?=<8Pl@ zD!9c*o3aRru#!VL2)@NjTbOuwJ7`39P1*usOIUH5Wt;R)e8U1s{pFUl7j1wuNv;82eOCjlM@2%nZ|!cDHzLU%*3Xb8X2iv zBg|a0iZ$nm4#;zS^=UeCpSND}HzdqIt&ea7_JX<<@hk7Sw*vFYL`_;iKt2eCd;4bmEq z{qH;LMyVv^(J!z2wUr_Xqu>;Ucz4>2Vm!CZ;VXp=H)P=UFM*BW8XMjiZotQ&QWv8( z9c2w=xqRGg*7R>5(3P6j_Y*oNrWPhi_X~I%m7Y$+2v?;6U5AWy#)@Sx+{q%!U7+B| zF+PMNe5x$O8QsMnzC^tFqIZ>$`IhXaX2ugY)$Wk?7M+?$`!|k4j6jtFV7TC_W<_ES z^%IeoiM&QZTFbt+fwzV9peXY^sf)YXI=@%sV_-~TPA{UhVifHKUOLnjlcSU6?59X4 z1>_nmc1WKiCfvucYMq&TGn*~j9h4>PrdeeolU*zd8yf<^9~MZMAz1>pL_7ipLv{X- zqqv14A+YPn_q{+_=T6(u=)}6Lpy_*$x8qbS^@pmXiQ_AKAlrLxItk>oyT1DxGa;$O zYpoZa|N8^lqwxiM#Oau*bjB$xd({^7Y%k`C_#a`e0IRaNz>Px4w&P2%hwj;+zdgQp zSH`iu)p;zV<&?+u$Tnv*d0R_u-Mw~UBa0sV;#sCv^j&_YKuRv&Ng;JMKNXZ2v%=0S z89Ag0kg(5ty%C^qTEINk$C8X!$MPI*|(C9j(z3YCN z)ttAZ!-@|yulEMoKt$PPh7~nkOH|}sN9lk=D;Lh`x1II>p) zA;%>20tS1_Z%&lP+koCl;IyCV&;I$Ugw2C_hXF{6l4k84t#_V!`L}C9j$y|pf!!%; zcUmjViz~_T#m_pr3yKJ>;NwND40z|lTerMDukczFB;^qwPBxV69mh29J%8gcpjRJV z7tbN$dc2hk^b@26+oli|i6>zf)O6QK;kF13p5fJ@h+we@U5sxu|ANO*Y2D&jIBzXRcLTPog_1bf^eIo0Nv5|*~9t1+;{D-}&f z978HvjaV2M2DV7Eng+WWfqb~YZt&%6(1fB+V16X9fVL*s-hrDWHMY$gb(XtHZ1iX_ zz`6XQ_Ob_O311ZYA+~w9-s4hQr}RM|K5}q^mLjq7**B+{9FLZX)LTJAgWXKLy4)`I z>R$%j%o-G~sC(n!k*u^)*ADJ>;vZuZ(%*`E;_y8`Y5m>>R3nO?rB}q|cqHV=()Jdk zvTWW`dhxX+exo*{d!a7rNfB;voSXlkI55P+NKX6ab2~BAy^dAK+>;XS63hQAt%IHU ze%-{){RXlcOH@H=O|zQYLWv6ITNu|~s}``XTC!E{Vq{;1PxCd3nCY#3cXBsKDjcrH$FpQ-M3nH-ghH0PWK;VbIQR%@t(W9#xn7>Ru&`eFL-P?f50?v?Bk@eUET<2tI> z(#;^($1h&fI>tQ5FOOYS6->>pj3|f3+M#Q#KO6Lx4~$vkg%0^mg%r*TW%&8*gU`(_ z1i1iR-nUeGU9ff4W2UapUDTP)q6V>~&oPehLND-#B-YwR zs%qIIY2++x&vLFED2f*IwPMS6?MO^x9k5%-qv)MIv=|i*c4ygn9()>dbT=Mc69PRg7VUK_ zMXIeC*wN8V&vO`Y+Sr7=Jv<4-H=u12^R65+>8~@~!(|Q+N7AQM8Da6=38D$git)sN(}SMJ(2x+ zf`1I+mlMc25;6Q(y){E?^3kzc@KPIu%OHIO>HSgL^Dwy-k(M4e$dyFfUK)j!3e76h zvHMWtGVv?9h3g&Yr|yS~mhwDa)9>zKDdE(3H0WXy>P4;ENDA%TSEd|qyY=-5FsJwX z?fOKQ&^_Op1NWEA7#r;0dQ@r2>|i9y&w?-ff1d|L8Rn8lT>6y>t_1i-cn!w^vt>+p zTLdO(Tf;W4^hkcmh&ZC8kF|JHvzK7*L~Fh0bKIFK^&*ysGCu837N-^p4I$RtigYRr zk`C@R`lAeY;)tMNd{g_pj5~VmYaqmua>3eq%Y}BNFJIIidxYDb3e7P>1uKGO7(!)C zd+W~PV+{sWj+M7c29;^J(7XOC@;#fjo`5x-^dA?u$91+|TyAfC+jRpyV0eT(?@PQp zyx>qeh2K`R4(Z#v6b<+x<fhi7K>nQxATCQ(Iu!p`7-JRrX#jX0 zFMN`Z{~O!@$SWcM;z8FTIgI~+=RXrFX)W4e41a?gcoMyK0OG%I!X<_O0ndL8@~=q# z*C786&;JjDT(n|jc$^```ifCF@A372$F&9O)2SHP;o%`FK8@e79|prGBwJzbJ%9hN z7(xcZ;jo@bM->nR*NwaSx8Dj|$z#|{BHdSZ?enUEaay>?oHQf-CF=0-M>`C8`RDi0qK4!b*n!79&Wq-=AR_*o-#s}o zvqzwR-1X@ciJPPz)StNXG;Twk5F`~uFip!4h_{s-Xfv+@j2>B7qy45Vc( z^vMxGQtDM$sFpAOR|?4nl9D<1ccOo>fGcVsDMhrnhUx#~srE_$NvU4#3&p=!0FD#L z*;k=c=SBaz8&@DH`6TEv|Ai|2`3icW4CHLwS^1I1zmoRUAC`3MgXG`I6bJs Date: Fri, 18 Mar 2022 10:45:54 +0100 Subject: [PATCH 13/23] Improving README --- examples/cloud-operations/network-dashboard/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md index 126651ca..20094144 100644 --- a/examples/cloud-operations/network-dashboard/README.md +++ b/examples/cloud-operations/network-dashboard/README.md @@ -7,6 +7,9 @@ Here is an example of dashboard you can get with this solution: +Here you see utilization (usage compared to the limit) for a specific metric (number of instances per VPC) for multiple VPCs and projects. +3 metrics are created: Usage, limit and utilization. You can follow each of these and create alerting policies if a threshold is reached. + ## Usage Clone this repository, then go through the following steps to create resources: From 344a489a27f9675dcfa5718511ad61bbb828f2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Fri, 18 Mar 2022 10:46:30 +0100 Subject: [PATCH 14/23] Improving README --- examples/cloud-operations/network-dashboard/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md index 20094144..ee3167cc 100644 --- a/examples/cloud-operations/network-dashboard/README.md +++ b/examples/cloud-operations/network-dashboard/README.md @@ -8,6 +8,7 @@ Here is an example of dashboard you can get with this solution: Here you see utilization (usage compared to the limit) for a specific metric (number of instances per VPC) for multiple VPCs and projects. + 3 metrics are created: Usage, limit and utilization. You can follow each of these and create alerting policies if a threshold is reached. ## Usage From 291470b8c4bebddbedae78c437a43c88d9e8faf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Thu, 24 Mar 2022 10:07:58 +0100 Subject: [PATCH 15/23] Refactoring to use Cloud Asset Inventory, reducing latency for metrics data, reduced execution time by 50%. --- .../network-dashboard/README.md | 1 - .../network-dashboard/cloud-function/main.py | 338 +++++++++++++----- .../cloud-function/requirements.txt | 3 +- .../network-dashboard/main.tf | 2 + .../network-dashboard/tests/test.tf | 1 - 5 files changed, 245 insertions(+), 100 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md index ee3167cc..3e47fadf 100644 --- a/examples/cloud-operations/network-dashboard/README.md +++ b/examples/cloud-operations/network-dashboard/README.md @@ -27,7 +27,6 @@ A dashboard called "quotas-utilization" should be created. The Cloud Function runs every 5 minutes by default so you should start getting some data points after a few minutes. You can change this frequency by modifying the "schedule_cron" variable in variables.tf. -Note that we are using Google defined metrics that are populated only once a day so you might need to wait up to one day for some metrics. Once done testing, you can clean up resources by running `terraform destroy`. diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 0b1e3dff..b374d879 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -18,9 +18,13 @@ import os import time import yaml from google.api import metric_pb2 as ga_metric -from google.cloud import monitoring_v3 +from google.api_core import protobuf_helpers +from google.cloud import monitoring_v3, asset_v1 +from google.protobuf import field_mask_pb2 from googleapiclient import discovery +# Organization ID containing the projects to be monitored +ORGANIZATION_ID = os.environ.get("ORGANIZATION_ID") # list of projects from which function will get quotas information MONITORED_PROJECTS_LIST = os.environ.get("MONITORED_PROJECTS_LIST").split(",") # project where the metrics and dahsboards will be created @@ -59,39 +63,192 @@ def main(event, context): Returns: 'Function executed successfully' ''' - metrics_dict = create_metrics() + # Asset inventory queries + gce_instance_dict = get_gce_instance_dict() + l4_forwarding_rules_dict = get_l4_forwarding_rules_dict() + l7_forwarding_rules_dict = get_l7_forwarding_rules_dict() + subnet_range_dict = get_subnet_ranges_dict() + # Per Network metrics - get_gce_instances_data(metrics_dict) - get_l4_forwarding_rules_data(metrics_dict) + get_gce_instances_data(metrics_dict, gce_instance_dict) + get_l4_forwarding_rules_data(metrics_dict, l4_forwarding_rules_dict) get_vpc_peering_data(metrics_dict) get_pgg_data( metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], - GCE_INSTANCES_USAGE_METRIC, GCE_INSTANCES_LIMIT_METRIC, - LIMIT_INSTANCES_PPG) + gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC, LIMIT_INSTANCES_PPG) get_pgg_data( metrics_dict["metrics_per_peering_group"] - ["l4_forwarding_rules_per_peering_group"], - L4_FORWARDING_RULES_USAGE_METRIC, L4_FORWARDING_RULES_LIMIT_METRIC, - LIMIT_L4_PPG) + ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict, + L4_FORWARDING_RULES_LIMIT_METRIC, LIMIT_L4_PPG) get_pgg_data( metrics_dict["metrics_per_peering_group"] - ["l7_forwarding_rules_per_peering_group"], - L7_FORWARDING_RULES_USAGE_METRIC, L7_FORWARDING_RULES_LIMIT_METRIC, - LIMIT_L7_PPG) + ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict, + L7_FORWARDING_RULES_LIMIT_METRIC, LIMIT_L7_PPG) get_pgg_data( metrics_dict["metrics_per_peering_group"] - ["subnet_ranges_per_peering_group"], SUBNET_RANGES_USAGE_METRIC, + ["subnet_ranges_per_peering_group"], subnet_range_dict, SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS) return 'Function executed successfully' +def get_l4_forwarding_rules_dict(): + ''' + Calls the Asset Inventory API to get all L4 Forwarding Rules under the GCP organization. + + Parameters: + None + Returns: + forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network. + ''' + client = asset_v1.AssetServiceClient() + + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + forwarding_rules_dict = {} + + response = client.search_all_resources( + request={ + "scope": f"organizations/{ORGANIZATION_ID}", + "asset_types": ["compute.googleapis.com/ForwardingRule"], + "read_mask": read_mask, + }) + for resource in response: + internal = False + network_link = "" + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "loadBalancingScheme": + internal = (field_value == "INTERNAL") + if field_name == "network": + network_link = field_value + if internal: + if network_link in forwarding_rules_dict: + forwarding_rules_dict[network_link] += 1 + else: + forwarding_rules_dict[network_link] = 1 + + return forwarding_rules_dict + + +def get_l7_forwarding_rules_dict(): + ''' + Calls the Asset Inventory API to get all L7 Forwarding Rules under the GCP organization. + + Parameters: + None + Returns: + forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network. + ''' + client = asset_v1.AssetServiceClient() + + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + forwarding_rules_dict = {} + + response = client.search_all_resources( + request={ + "scope": f"organizations/{ORGANIZATION_ID}", + "asset_types": ["compute.googleapis.com/ForwardingRule"], + "read_mask": read_mask, + }) + for resource in response: + internal = False + network_link = "" + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "loadBalancingScheme": + internal = (field_value == "INTERNAL_MANAGED") + if field_name == "network": + network_link = field_value + if internal: + if network_link in forwarding_rules_dict: + forwarding_rules_dict[network_link] += 1 + else: + forwarding_rules_dict[network_link] = 1 + + return forwarding_rules_dict + + +def get_gce_instance_dict(): + ''' + Calls the Asset Inventory API to get all GCE instances under the GCP organization. + + Parameters: + None + Returns: + gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network. + ''' + client = asset_v1.AssetServiceClient() + + gce_instance_dict = {} + + response = client.search_all_resources( + request={ + "scope": f"organizations/{ORGANIZATION_ID}", + "asset_types": ["compute.googleapis.com/Instance"], + }) + for resource in response: + for field_name, field_value in resource.additional_attributes.items(): + if field_name == "networkInterfaceNetworks": + for network in field_value: + if network in gce_instance_dict: + gce_instance_dict[network] += 1 + else: + gce_instance_dict[network] = 1 + + return gce_instance_dict + + +def get_subnet_ranges_dict(): + ''' + Calls the Asset Inventory API to get all Subnet ranges under the GCP organization. + + Parameters: + None + Returns: + subnet_range_dict (dictionary of string: int): Keys are the network links and values are the number of subnet ranges per network. + ''' + client = asset_v1.AssetServiceClient() + subnet_range_dict = {} + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + response = client.search_all_resources( + request={ + "scope": f"organizations/{ORGANIZATION_ID}", + "asset_types": ["compute.googleapis.com/Subnetwork"], + "read_mask": read_mask, + }) + for resource in response: + ranges = 0 + network_link = None + + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "network": + network_link = field_value + ranges += 1 + if field_name == "secondaryIpRanges": + for range in field_value: + ranges += 1 + + if network_link in subnet_range_dict: + subnet_range_dict[network_link] += ranges + else: + subnet_range_dict[network_link] = ranges + + return subnet_range_dict + + def create_client(): ''' Creates the monitoring API client, that will be used to create, read and update custom metrics. @@ -123,6 +280,11 @@ def create_client(): def create_metrics(): + client = monitoring_v3.MetricServiceClient() + existing_metrics = [] + for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): + existing_metrics.append(desc.type) + with open("metrics.yaml", 'r') as stream: try: metrics_dict = yaml.safe_load(stream) @@ -130,7 +292,10 @@ def create_metrics(): for metric_list in metrics_dict.values(): for metric in metric_list.values(): for sub_metric in metric.values(): - create_metric(sub_metric["name"], sub_metric["description"]) + metric_link = f"custom.googleapis.com/{sub_metric['name']}" + # If the metric doesn't exist yet, then we create it + if metric_link not in existing_metrics: + create_metric(sub_metric["name"], sub_metric["description"]) return metrics_dict except yaml.YAMLError as exc: @@ -150,60 +315,54 @@ def create_metric(metric_name, description): ''' client = monitoring_v3.MetricServiceClient() - metric_link = f"custom.googleapis.com/{metric_name}" - types = [] - for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): - types.append(desc.type) - - # If the metric doesn't exist yet, then we create it - if metric_link not in types: - descriptor = ga_metric.MetricDescriptor() - descriptor.type = f"custom.googleapis.com/{metric_name}" - descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE - descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE - descriptor.description = description - descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, - metric_descriptor=descriptor) - print("Created {}.".format(descriptor.name)) + descriptor = ga_metric.MetricDescriptor() + descriptor.type = f"custom.googleapis.com/{metric_name}" + descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE + descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE + descriptor.description = description + descriptor = client.create_metric_descriptor(name=MONITORING_PROJECT_LINK, + metric_descriptor=descriptor) + print("Created {}.".format(descriptor.name)) -def get_gce_instances_data(metrics_dict): +def get_gce_instances_data(metrics_dict, gce_instance_dict): ''' Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. Parameters: metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions + gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network. Returns: - None + gce_instance_dict ''' # Existing GCP Monitoring metrics for GCE instances - metric_instances_usage = "compute.googleapis.com/quota/instances_per_vpc_network/usage" metric_instances_limit = "compute.googleapis.com/quota/instances_per_vpc_network/limit" for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_usage = get_quota_current_usage(f"projects/{project}", - metric_instances_usage) current_quota_limit = get_quota_current_limit(f"projects/{project}", metric_instances_limit) - - current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, - LIMIT_INSTANCES) + set_limits(net, current_quota_limit_view, LIMIT_INSTANCES) + + network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network name']}" + + usage = 0 + if network_link in gce_instance_dict: + usage = gce_instance_dict[network_link] + write_data_to_metric( - project, net['usage'], metrics_dict["metrics_per_network"] + project, usage, metrics_dict["metrics_per_network"] ["instance_per_network"]["usage"]["name"], net['network name']) write_data_to_metric( project, net['limit'], metrics_dict["metrics_per_network"] ["instance_per_network"]["limit"]["name"], net['network name']) write_data_to_metric( - project, net['usage'] / net['limit'], - metrics_dict["metrics_per_network"]["instance_per_network"] - ["utilization"]["name"], net['network name']) + project, usage / net['limit'], metrics_dict["metrics_per_network"] + ["instance_per_network"]["utilization"]["name"], net['network name']) print(f"Wrote number of instances to metric for projects/{project}") @@ -316,35 +475,38 @@ def get_limit(network_name, limit_list): return 0 -def get_l4_forwarding_rules_data(metrics_dict): +def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict): ''' Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. Parameters: metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions + forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network. Returns: None ''' # Existing GCP Monitoring metrics for L4 Forwarding Rules - l4_forwarding_rules_usage = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_usage = get_quota_current_usage(f"projects/{project}", - l4_forwarding_rules_usage) current_quota_limit = get_quota_current_limit(f"projects/{project}", l4_forwarding_rules_limit) - current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_usage_limits(net, current_quota_usage_view, current_quota_limit_view, - LIMIT_L4) + set_limits(net, current_quota_limit_view, LIMIT_L4) + + network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network name']}" + + usage = 0 + if network_link in forwarding_rules_dict: + usage = forwarding_rules_dict[network_link] + write_data_to_metric( - project, net['usage'], metrics_dict["metrics_per_network"] + project, usage, metrics_dict["metrics_per_network"] ["l4_forwarding_rules_per_network"]["usage"]["name"], net['network name']) write_data_to_metric( @@ -352,22 +514,22 @@ def get_l4_forwarding_rules_data(metrics_dict): ["l4_forwarding_rules_per_network"]["limit"]["name"], net['network name']) write_data_to_metric( - project, net['usage'] / net['limit'], - metrics_dict["metrics_per_network"]["l4_forwarding_rules_per_network"] - ["utilization"]["name"], net['network name']) + project, usage / net['limit'], metrics_dict["metrics_per_network"] + ["l4_forwarding_rules_per_network"]["utilization"]["name"], + net['network name']) print( f"Wrote number of L4 forwarding rules to metric for projects/{project}") -def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): +def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_ppg): ''' This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored. Parameters: - metric_dict (dictionary of string: string): A dictionary with the metric names and description, that will be used to populate the metrics + metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics usage_metric (string): Name of the existing GCP metric for usage per VPC network. - limit_metric (string): Name of the existing GCP metric for limit per VPC network. + usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit). Returns: None @@ -382,34 +544,38 @@ def get_pgg_data(metric_dict, usage_metric, limit_metric, limit_ppg): # For each network in this GCP project for network_dict in network_dict_list: - current_quota_usage = get_quota_current_usage(f"projects/{project}", - usage_metric) current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) - - current_quota_usage_view = customize_quota_view(current_quota_usage) current_quota_limit_view = customize_quota_view(current_quota_limit) + limit = get_limit_values(network_dict, current_quota_limit_view, + limit_ppg) + + network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}" + + usage = 0 + if network_link in usage_dict: + usage = usage_dict[network_link] - usage, limit = get_usage_limit(network_dict, current_quota_usage_view, - current_quota_limit_view, limit_ppg) # Here we add usage and limit to the network dictionary network_dict["usage"] = usage network_dict["limit"] = limit # For every peered network, get usage and limits for peered_network in network_dict['peerings']: - peering_project_usage = customize_quota_view( - get_quota_current_usage(f"projects/{peered_network['project_id']}", - usage_metric)) + peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}" + peered_usage = 0 + if peered_network_link in usage_dict: + peered_usage = usage_dict[peered_network_link] + peering_project_limit = customize_quota_view( get_quota_current_limit(f"projects/{peered_network['project_id']}", limit_metric)) - usage, limit = get_usage_limit(peered_network, peering_project_usage, - peering_project_limit, limit_ppg) + peered_limit = get_limit_values(peered_network, peering_project_limit, + limit_ppg) # Here we add usage and limit to the peered network dictionary - peered_network["usage"] = usage - peered_network["limit"] = limit + peered_network["usage"] = peered_usage + peered_network["limit"] = peered_limit count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], @@ -630,29 +796,17 @@ def customize_quota_view(quota_results): return quotaViewList -def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): +def set_limits(network_dict, quota_limit, limit_list): ''' - Updates the network dictionary with quota usage and limit values. + Updates the network dictionary with quota limit values. Parameters: network_dict (dictionary of string: string): Contains network information. - quota_usage (list of dictionaries of string: string): Current quota usage. quota_limit (list of dictionaries of string: string): Current quota limit. limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit). Returns: None ''' - if quota_usage: - for net in quota_usage: - if net['network_id'] == network_dict[ - 'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network_dict['usage'] = net['value'] # set network usage in dictionary - break - else: - network_dict['usage'] = 0 # if network does not appear in GCP quotas - else: - network_dict['usage'] = 0 # if quotas does not appear in GCP quotas - if quota_limit: for net in quota_limit: if net['network_id'] == network_dict[ @@ -678,29 +832,19 @@ def set_usage_limits(network_dict, quota_usage, quota_limit, limit_list): 1]) -def get_usage_limit(network, quota_usage, quota_limit, limit_list): +def get_limit_values(network, quota_limit, limit_list): ''' - Returns usage and limit for a specific network and metric. + Returns uslimit for a specific network and metric. Parameters: network_dict (dictionary of string: string): Contains network information. - quota_usage (list of dictionaries of string: string): Current quota usage for all networks in that project. quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project. limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit). Returns: - usage (int): Current usage for that network. - limit (int): Current usage for that network. + limit (int): Current limit for that network. ''' - usage = 0 limit = 0 - if quota_usage: - for net in quota_usage: - if net['network_id'] == network[ - 'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - usage = net['value'] # set network usage in dictionary - break - if quota_limit: for net in quota_limit: if net['network_id'] == network[ @@ -721,7 +865,7 @@ def get_usage_limit(network, quota_usage, quota_limit, limit_list): else: limit = int(limit_list[limit_list.index('default_value') + 1]) - return usage, limit + return limit def write_data_to_metric(monitored_project_id, value, metric_name, diff --git a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt index 0888969c..8a6a5960 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt +++ b/examples/cloud-operations/network-dashboard/cloud-function/requirements.txt @@ -6,4 +6,5 @@ google-cloud-logging==3.0.0 google-cloud-monitoring==2.9.1 oauth2client==4.1.3 google-api-core==2.7.0 -PyYAML==6.0 \ No newline at end of file +PyYAML==6.0 +google-cloud-asset==3.8.1 \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 152bb360..343f3c31 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -66,6 +66,7 @@ module "service-account-function" { "${var.organization_id}" = [ "roles/compute.networkViewer", "roles/monitoring.viewer", + "roles/cloudasset.viewer" ] } @@ -139,6 +140,7 @@ module "cloud-function" { LIMIT_VPC_PEER = local.limit_vpc_peer MONITORED_PROJECTS_LIST = local.projects MONITORING_PROJECT_ID = module.project-monitoring.project_id + ORGANIZATION_ID = var.organization_id } service_account = module.service-account-function.email diff --git a/examples/cloud-operations/network-dashboard/tests/test.tf b/examples/cloud-operations/network-dashboard/tests/test.tf index 8161af65..791d68b0 100644 --- a/examples/cloud-operations/network-dashboard/tests/test.tf +++ b/examples/cloud-operations/network-dashboard/tests/test.tf @@ -270,7 +270,6 @@ resource "google_compute_instance" "test-vm-hub1" { } # Forwarding Rules - resource "google_compute_forwarding_rule" "forwarding-rule-dev" { count = 10 name = "forwarding-rule-dev${count.index}" From 02dc53d0d433d931ca254e6496b488dd5760570f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Mon, 28 Mar 2022 18:44:16 +0200 Subject: [PATCH 16/23] Refactored how limits are managed, now you can edit the metrics.yaml file to set specific metrics per network. --- .../network-dashboard/README.md | 11 +- .../network-dashboard/cloud-function/main.py | 302 +++++++++--------- .../cloud-function/metrics.yaml | 25 +- .../dashboards/quotas-utilization.json | 4 +- .../network-dashboard/main.tf | 25 -- .../network-dashboard/variables.tf | 65 ---- 6 files changed, 184 insertions(+), 248 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md index 3e47fadf..2f958375 100644 --- a/examples/cloud-operations/network-dashboard/README.md +++ b/examples/cloud-operations/network-dashboard/README.md @@ -42,4 +42,13 @@ The Cloud Function currently tracks usage, limit and utilization of: - internal forwarding rules for internal L4 load balancers per VPC peering group - internal forwarding rules for internal L7 load balancers per VPC peering group -It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. \ No newline at end of file +It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. + +## Next steps and ideas +In a future release, we could support: +- Static routes per VPC / per VPC peering group +- Dynamic routes per VPC / per VPC peering group +- Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) +- Subnet IP ranges utilization + +If you are interested in this and/or would like to contribute, please contact legranda@google.com. \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index b374d879..6f7e2135 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -17,6 +17,7 @@ import os import time import yaml +from collections import defaultdict from google.api import metric_pb2 as ga_metric from google.api_core import protobuf_helpers from google.cloud import monitoring_v3, asset_v1 @@ -32,25 +33,11 @@ MONITORING_PROJECT_ID = os.environ.get("MONITORING_PROJECT_ID") MONITORING_PROJECT_LINK = f"projects/{MONITORING_PROJECT_ID}" service = discovery.build('compute', 'v1') -# DEFAULT LIMITS: -LIMIT_INSTANCES = os.environ.get("LIMIT_INSTANCES").split(",") -LIMIT_INSTANCES_PPG = os.environ.get("LIMIT_INSTANCES_PPG").split(",") -LIMIT_L4 = os.environ.get("LIMIT_L4").split(",") -LIMIT_L4_PPG = os.environ.get("LIMIT_L4_PPG").split(",") -LIMIT_L7 = os.environ.get("LIMIT_L7").split(",") -LIMIT_L7_PPG = os.environ.get("LIMIT_L7_PPG").split(",") -LIMIT_SUBNETS = os.environ.get("LIMIT_SUBNETS").split(",") -LIMIT_VPC_PEER = os.environ.get("LIMIT_VPC_PEER").split(",") - # Existing GCP metrics per network GCE_INSTANCES_LIMIT_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/limit" -GCE_INSTANCES_USAGE_METRIC = "compute.googleapis.com/quota/instances_per_vpc_network/usage" L4_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" -L4_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/usage" L7_FORWARDING_RULES_LIMIT_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit" -L7_FORWARDING_RULES_USAGE_METRIC = "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/usage" SUBNET_RANGES_LIMIT_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" -SUBNET_RANGES_USAGE_METRIC = "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/usage" def main(event, context): @@ -63,7 +50,7 @@ def main(event, context): Returns: 'Function executed successfully' ''' - metrics_dict = create_metrics() + metrics_dict, limits_dict = create_metrics() # Asset inventory queries gce_instance_dict = get_gce_instance_dict() @@ -72,28 +59,28 @@ def main(event, context): subnet_range_dict = get_subnet_ranges_dict() # Per Network metrics - get_gce_instances_data(metrics_dict, gce_instance_dict) - get_l4_forwarding_rules_data(metrics_dict, l4_forwarding_rules_dict) - get_vpc_peering_data(metrics_dict) + get_gce_instances_data(metrics_dict, gce_instance_dict, limits_dict['number_of_instances_limit']) + get_l4_forwarding_rules_data(metrics_dict, l4_forwarding_rules_dict, limits_dict['internal_forwarding_rules_l4_limit']) + get_vpc_peering_data(metrics_dict, limits_dict['number_of_vpc_peerings_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], - gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC, LIMIT_INSTANCES_PPG) + gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC, limits_dict['number_of_instances_ppg_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"] ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict, - L4_FORWARDING_RULES_LIMIT_METRIC, LIMIT_L4_PPG) + L4_FORWARDING_RULES_LIMIT_METRIC, limits_dict['internal_forwarding_rules_l4_ppg_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"] ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict, - L7_FORWARDING_RULES_LIMIT_METRIC, LIMIT_L7_PPG) + L7_FORWARDING_RULES_LIMIT_METRIC, limits_dict['internal_forwarding_rules_l7_ppg_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"] ["subnet_ranges_per_peering_group"], subnet_range_dict, - SUBNET_RANGES_LIMIT_METRIC, LIMIT_SUBNETS) + SUBNET_RANGES_LIMIT_METRIC, limits_dict['number_of_subnet_IP_ranges_limit']) return 'Function executed successfully' @@ -112,7 +99,7 @@ def get_l4_forwarding_rules_dict(): read_mask = field_mask_pb2.FieldMask() read_mask.FromJsonString('name,versionedResources') - forwarding_rules_dict = {} + forwarding_rules_dict = defaultdict(int) response = client.search_all_resources( request={ @@ -152,7 +139,7 @@ def get_l7_forwarding_rules_dict(): read_mask = field_mask_pb2.FieldMask() read_mask.FromJsonString('name,versionedResources') - forwarding_rules_dict = {} + forwarding_rules_dict = defaultdict(int) response = client.search_all_resources( request={ @@ -189,7 +176,7 @@ def get_gce_instance_dict(): ''' client = asset_v1.AssetServiceClient() - gce_instance_dict = {} + gce_instance_dict = defaultdict(int) response = client.search_all_resources( request={ @@ -218,7 +205,7 @@ def get_subnet_ranges_dict(): subnet_range_dict (dictionary of string: int): Keys are the network links and values are the number of subnet ranges per network. ''' client = asset_v1.AssetServiceClient() - subnet_range_dict = {} + subnet_range_dict = defaultdict(int) read_mask = field_mask_pb2.FieldMask() read_mask.FromJsonString('name,versionedResources') @@ -280,10 +267,21 @@ def create_client(): def create_metrics(): + ''' + Creates all Cloud Monitoring custom metrics based on the metric.yaml file + + Parameters: + None + + Returns: + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions + limits_dict (dictionary of dictionary of string: int): limits_dict[metric_name]: dict[network_name] = limit_value + ''' client = monitoring_v3.MetricServiceClient() existing_metrics = [] for desc in client.list_metric_descriptors(name=MONITORING_PROJECT_LINK): existing_metrics.append(desc.type) + limits_dict = {} with open("metrics.yaml", 'r') as stream: try: @@ -291,13 +289,19 @@ def create_metrics(): for metric_list in metrics_dict.values(): for metric in metric_list.values(): - for sub_metric in metric.values(): + for sub_metric_key, sub_metric in metric.items(): metric_link = f"custom.googleapis.com/{sub_metric['name']}" # If the metric doesn't exist yet, then we create it if metric_link not in existing_metrics: create_metric(sub_metric["name"], sub_metric["description"]) - - return metrics_dict + # Parse limits (both default values and network specific ones) + if sub_metric_key == "limit": + limits_dict_for_metric = {} + for network_link, limit_value in sub_metric["values"].items(): + limits_dict_for_metric[network_link] = limit_value + limits_dict[sub_metric["name"]] = limits_dict_for_metric + + return metrics_dict, limits_dict except yaml.YAMLError as exc: print(exc) @@ -325,13 +329,14 @@ def create_metric(metric_name, description): print("Created {}.".format(descriptor.name)) -def get_gce_instances_data(metrics_dict, gce_instance_dict): +def get_gce_instances_data(metrics_dict, gce_instance_dict, limit_dict): ''' Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. Parameters: metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value Returns: gce_instance_dict ''' @@ -346,9 +351,9 @@ def get_gce_instances_data(metrics_dict, gce_instance_dict): current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_limits(net, current_quota_limit_view, LIMIT_INSTANCES) + set_limits(net, current_quota_limit_view, limit_dict) - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network name']}" + network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network_name']}" usage = 0 if network_link in gce_instance_dict: @@ -356,29 +361,29 @@ def get_gce_instances_data(metrics_dict, gce_instance_dict): write_data_to_metric( project, usage, metrics_dict["metrics_per_network"] - ["instance_per_network"]["usage"]["name"], net['network name']) + ["instance_per_network"]["usage"]["name"], net['network_name']) write_data_to_metric( project, net['limit'], metrics_dict["metrics_per_network"] - ["instance_per_network"]["limit"]["name"], net['network name']) + ["instance_per_network"]["limit"]["name"], net['network_name']) write_data_to_metric( project, usage / net['limit'], metrics_dict["metrics_per_network"] - ["instance_per_network"]["utilization"]["name"], net['network name']) + ["instance_per_network"]["utilization"]["name"], net['network_name']) print(f"Wrote number of instances to metric for projects/{project}") -def get_vpc_peering_data(metrics_dict): +def get_vpc_peering_data(metrics_dict, limit_dict): ''' Gets the data for VPC peerings (active or not) and writes it to the metric defined (vpc_peering_active_metric and vpc_peering_metric). Parameters: metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value Returns: None ''' for project in MONITORED_PROJECTS_LIST: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data( - project, LIMIT_VPC_PEER) + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_dict) for peering in active_vpc_peerings: write_data_to_metric( project, peering['active_peerings'], @@ -409,13 +414,13 @@ def get_vpc_peering_data(metrics_dict): print("Wrote number of VPC peerings to custom metric for project:", project) -def gather_vpc_peerings_data(project_id, limit_list): +def gather_vpc_peerings_data(project_id, limit_dict): ''' Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. Parameters: project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs. - limit_list (list of string): Used to get the limit per VPC or the default limit. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value Returns: active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering. peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering. @@ -437,69 +442,71 @@ def gather_vpc_peerings_data(project_id, limit_list): else: peerings_count = 0 active_peerings_count = 0 + + network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network['name']}" + network_limit = get_limit_ppg(network_link, limit_dict) active_d = { 'project_id': project_id, 'network_name': network['name'], 'active_peerings': active_peerings_count, - 'network_limit': get_limit(network['name'], limit_list) + 'network_limit': network_limit } active_peerings_dict.append(active_d) d = { 'project_id': project_id, 'network_name': network['name'], 'peerings': peerings_count, - 'network_limit': get_limit(network['name'], limit_list) + 'network_limit': network_limit } peerings_dict.append(d) return active_peerings_dict, peerings_dict -def get_limit(network_name, limit_list): +def get_limit_ppg(network_link, limit_dict): ''' Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. Parameters: - network_name (string): Name of the VPC network. + network_link (string): VPC network link. limit_list (list of string): Used to get the limit per VPC or the default limit. Returns: - limit (int): Limit for that VPC and that metric. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value ''' - if network_name in limit_list: - return int(limit_list[limit_list.index(network_name) + 1]) + if network_link in limit_dict: + return limit_dict[network_link] else: - if 'default_value' in limit_list: - return int(limit_list[limit_list.index('default_value') + 1]) + if 'default_value' in limit_dict: + return limit_dict['default_value'] else: + print(f"Error: limit not found for {network_link}") return 0 -def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict): +def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict, limit_dict): ''' Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. Parameters: - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. Returns: None ''' - # Existing GCP Monitoring metrics for L4 Forwarding Rules - l4_forwarding_rules_limit = "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit" - for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) current_quota_limit = get_quota_current_limit(f"projects/{project}", - l4_forwarding_rules_limit) + L4_FORWARDING_RULES_LIMIT_METRIC) current_quota_limit_view = customize_quota_view(current_quota_limit) for net in network_dict: - set_limits(net, current_quota_limit_view, LIMIT_L4) + set_limits(net, current_quota_limit_view, limit_dict) - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network name']}" + network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network_name']}" usage = 0 if network_link in forwarding_rules_dict: @@ -508,21 +515,21 @@ def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict): write_data_to_metric( project, usage, metrics_dict["metrics_per_network"] ["l4_forwarding_rules_per_network"]["usage"]["name"], - net['network name']) + net['network_name']) write_data_to_metric( project, net['limit'], metrics_dict["metrics_per_network"] ["l4_forwarding_rules_per_network"]["limit"]["name"], - net['network name']) + net['network_name']) write_data_to_metric( project, usage / net['limit'], metrics_dict["metrics_per_network"] ["l4_forwarding_rules_per_network"]["utilization"]["name"], - net['network name']) + net['network_name']) print( f"Wrote number of L4 forwarding rules to metric for projects/{project}") -def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_ppg): +def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_dict): ''' This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored. @@ -530,7 +537,7 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_ppg): metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics usage_metric (string): Name of the existing GCP metric for usage per VPC network. usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value - limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit). + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value Returns: None ''' @@ -544,13 +551,13 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_ppg): # For each network in this GCP project for network_dict in network_dict_list: + network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}" + current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) current_quota_limit_view = customize_quota_view(current_quota_limit) - limit = get_limit_values(network_dict, current_quota_limit_view, - limit_ppg) - - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network_dict['network_name']}" + limit = get_limit_network(network_dict, network_link, current_quota_limit_view, + limit_dict) usage = 0 if network_link in usage_dict: @@ -561,25 +568,25 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_ppg): network_dict["limit"] = limit # For every peered network, get usage and limits - for peered_network in network_dict['peerings']: - peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}" + for peered_network_dict in network_dict['peerings']: + peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}" peered_usage = 0 if peered_network_link in usage_dict: peered_usage = usage_dict[peered_network_link] peering_project_limit = customize_quota_view( - get_quota_current_limit(f"projects/{peered_network['project_id']}", + get_quota_current_limit(f"projects/{peered_network_dict['project_id']}", limit_metric)) - peered_limit = get_limit_values(peered_network, peering_project_limit, - limit_ppg) + peered_limit = get_limit_network(peered_network_dict, peered_network_link, peering_project_limit, + limit_dict) # Here we add usage and limit to the peered network dictionary - peered_network["usage"] = peered_usage - peered_network["limit"] = peered_limit + peered_network_dict["usage"] = peered_usage + peered_network_dict["limit"] = peered_limit count_effective_limit(project, network_dict, metric_dict["usage"]["name"], metric_dict["limit"]["name"], - metric_dict["utilization"]["name"], limit_ppg) + metric_dict["utilization"]["name"], limit_dict) print( f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}" ) @@ -587,7 +594,7 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_ppg): def count_effective_limit(project_id, network_dict, usage_metric_name, limit_metric_name, utilization_metric_name, - limit_ppg): + limit_dict): ''' Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit @@ -598,7 +605,7 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group. limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group. utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group. - limit_ppg (list of string): List containing the limit per peering group (either VPC specific or default limit). + limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit). Returns: None ''' @@ -611,16 +618,19 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, for peered_network in network_dict['peerings']: peering_group_usage += peered_network['usage'] + network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" + # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) - limit_step1 = max(network_dict['limit'], - get_limit(network_dict['network_name'], limit_ppg)) + limit_step1 = max(network_dict['limit'], get_limit_ppg(network_link, limit_dict)) # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network limit_step2 = [] for peered_network in network_dict['peerings']: + peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}" + limit_step2.append( max(peered_network['limit'], - get_limit(peered_network['network_name'], limit_ppg))) + get_limit_ppg(peered_network_link, limit_dict))) # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 limit_step3 = min(limit_step2) @@ -636,7 +646,6 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) - def get_networks(project_id): ''' Returns a dictionary of all networks in a project. @@ -651,12 +660,32 @@ def get_networks(project_id): network_dict = [] if 'items' in response: for network in response['items']: - NETWORK = network['name'] - ID = network['id'] - d = {'project_id': project_id, 'network name': NETWORK, 'network id': ID} + network_name = network['name'] + network_id = network['id'] + d = {'project_id': project_id, 'network_name': network_name, 'network_id': network_id} network_dict.append(d) return network_dict +# TODO: list all routers (https://cloud.google.com/compute/docs/reference/rest/v1/routers/list) then https://cloud.google.com/compute/docs/reference/rest/v1/routers/getRouterStatus +def get_routes(project_id): + ''' + Returns a dictionary of all dynamic routes in a project. + + Parameters: + project_id (string): Project ID for the project containing the networks. + Returns: + network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) + ''' + request = service.routers().list(project=project_id) + response = request.execute() + network_dict = [] + if 'items' in response: + for router in response['items']: + network_name = router['name'] + network_id = router['id'] + d = {'project_id': project_id, 'network name': network_name, 'network id': network_id} + network_dict.append(d) + return network_dict def gather_peering_data(project_id): ''' @@ -729,30 +758,6 @@ def get_network_id(project_id, network_name): return network_id - -def get_quota_current_usage(project_link, metric_name): - ''' - Retrieves quota usage for a specific metric. - - Parameters: - project_link (string): Project link. - metric_name (string): Name of the metric. - Returns: - results_list (list of string): Current usage. - ''' - client, interval = create_client() - - results = client.list_time_series( - request={ - "name": project_link, - "filter": f'metric.type = "{metric_name}"', - "interval": interval, - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) - results_list = list(results) - return (results_list) - - def get_quota_current_limit(project_link, metric_name): ''' Retrieves limit for a specific metric. @@ -796,76 +801,64 @@ def customize_quota_view(quota_results): return quotaViewList -def set_limits(network_dict, quota_limit, limit_list): +def set_limits(network_dict, quota_limit, limit_dict): ''' Updates the network dictionary with quota limit values. Parameters: network_dict (dictionary of string: string): Contains network information. quota_limit (list of dictionaries of string: string): Current quota limit. - limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit). + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value Returns: None ''' + + network_dict['limit'] = None + if quota_limit: for net in quota_limit: if net['network_id'] == network_dict[ - 'network id']: # if network ids in GCP quotas and in dictionary (using API) are the same - network_dict['limit'] = net['value'] # set network limit in dictionary - break - else: - if network_dict[ - 'network name'] in limit_list: # if network limit is in the environmental variables - network_dict['limit'] = int( - limit_list[limit_list.index(network_dict['network name']) + 1]) - else: - network_dict['limit'] = int( - limit_list[limit_list.index('default_value') + - 1]) # set default value - else: # if quotas does not appear in GCP quotas - if network_dict['network name'] in limit_list: - network_dict['limit'] = int( - limit_list[limit_list.index(network_dict['network name']) + - 1]) # ["default", 100, "networkname", 200] + 'network_id']: + network_dict['limit'] = net['value'] + return + + network_link = f"https://www.googleapis.com/compute/v1/projects/{network_dict['project_id']}/global/networks/{network_dict['network_name']}" + + if network_link in limit_dict: + network_dict['limit'] = limit_dict[network_link] + else: + if 'default_value' in limit_dict: + network_dict['limit'] = limit_dict['default_value'] else: - network_dict['limit'] = int(limit_list[limit_list.index('default_value') + - 1]) + print(f"Error: Couldn't find limit for {network_link}") + network_dict['limit'] = 0 - -def get_limit_values(network, quota_limit, limit_list): +def get_limit_network(network_dict, network_link, quota_limit, limit_dict): ''' - Returns uslimit for a specific network and metric. + Returns limit for a specific network and metric, using the GCP quota metrics or the values in the yaml file if not found. Parameters: network_dict (dictionary of string: string): Contains network information. + network_link (string): Contains network link quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project. - limit_list (list of string): List containing the limit per VPC (either VPC specific or default limit). + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value Returns: limit (int): Current limit for that network. ''' - limit = 0 - if quota_limit: for net in quota_limit: - if net['network_id'] == network[ - 'network_id']: # if network ids in GCP quotas and in dictionary (using API) are the same - limit = net['value'] # set network limit in dictionary - break - else: - if network[ - 'network_name'] in limit_list: # if network limit is in the environmental variables - limit = int(limit_list[limit_list.index(network['network_name']) + 1]) - else: - limit = int(limit_list[limit_list.index('default_value') + - 1]) # set default value - else: # if quotas does not appear in GCP quotas - if network['network_name'] in limit_list: - limit = int(limit_list[limit_list.index(network['network_name']) + - 1]) # ["default", 100, "networkname", 200] + if net['network_id'] == network_dict['network_id']: + return net['value'] + + if network_link in limit_dict: + return limit_dict[network_link] + else: + if 'default_value' in limit_dict: + return limit_dict['default_value'] else: - limit = int(limit_list[limit_list.index('default_value') + 1]) + print(f"Error: Couldn't find limit for {network_link}") - return limit + return 0 def write_data_to_metric(monitored_project_id, value, metric_name, @@ -906,4 +899,9 @@ def write_data_to_metric(monitored_project_id, value, metric_name, }) series.points = [point] - client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) + # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error + # Implement exponential backoff retries? + try: + client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) + except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 233dc9be..a9772b5c 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -22,6 +22,8 @@ metrics_per_network: limit: name: number_of_instances_limit description: Number of instances per VPC network - limit. + values: + default_value: 15000 utilization: name: number_of_instances_utilization description: Number of instances per VPC network - utilization. @@ -32,6 +34,8 @@ metrics_per_network: limit: name: number_of_active_vpc_peerings_limit description: Number of active VPC Peerings per VPC - limit. + values: + default_value: 25 utilization: name: number_of_active_vpc_peerings_utilization description: Number of active VPC Peerings per VPC - utilization. @@ -42,6 +46,9 @@ metrics_per_network: limit: name: number_of_vpc_peerings_limit description: Number of VPC Peerings per VPC - limit. + values: + default_value: 25 + https://www.googleapis.com/compute/v1/projects/net-dash-test-host-prod/global/networks/vpc-prod: 40 utilization: name: number_of_vpc_peerings_utilization description: Number of VPC Peerings per VPC - utilization. @@ -52,6 +59,8 @@ metrics_per_network: limit: name: internal_forwarding_rules_l4_limit description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - limit. + values: + default_value: 75 utilization: name: internal_forwarding_rules_l4_utilization description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization. @@ -62,6 +71,8 @@ metrics_per_network: limit: name: internal_forwarding_rules_l7_limit description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per network - effective limit. + values: + default_value: 75 utilization: name: internal_forwarding_rules_l7_utilization description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per Vnetwork - utilization. @@ -73,6 +84,8 @@ metrics_per_peering_group: limit: name: internal_forwarding_rules_l4_ppg_limit description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - effective limit. + values: + default_value: 175 utilization: name: internal_forwarding_rules_l4_ppg_utilization description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - utilization. @@ -83,18 +96,22 @@ metrics_per_peering_group: limit: name: internal_forwarding_rules_l7_ppg_limit description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - effective limit. + values: + default_value: 175 utilization: name: internal_forwarding_rules_l7_ppg_utilization description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - utilization. subnet_ranges_per_peering_group: usage: - name: number_of_subnet_IP_ranges_usage + name: number_of_subnet_IP_ranges_ppg_usage description: Number of Subnet Ranges per peering group - usage. limit: - name: number_of_subnet_IP_ranges_limit + name: number_of_subnet_IP_ranges_ppg_limit description: Number of Subnet Ranges per peering group - effective limit. + values: + default_value: 400 utilization: - name: number_of_subnet_IP_ranges_utilization + name: number_of_subnet_IP_ranges_ppg_utilization description: Number of Subnet Ranges per peering group - utilization. instance_per_peering_group: usage: @@ -103,6 +120,8 @@ metrics_per_peering_group: limit: name: number_of_instances_ppg_limit description: Number of instances per peering group - limit. + values: + default_value: 15500 utilization: name: number_of_instances_ppg_utilization description: Number of instances per peering group - utilization. \ No newline at end of file diff --git a/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index af812061..794d6123 100644 --- a/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -196,7 +196,7 @@ { "height": 4, "widget": { - "title": "number_of_subnet_IP_ranges_utilization", + "title": "number_of_subnet_IP_ranges_ppg_utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -212,7 +212,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_MEAN" diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 343f3c31..8fb7963c 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -17,23 +17,6 @@ locals { project_id_list = toset(var.monitored_projects_list) projects = join(",", local.project_id_list) - - limit_instances = join(",", local.limit_instances_list) - limit_instances_list = tolist(var.limit_instances) - limit_instances_ppg = join(",", local.limit_instances_ppg_list) - limit_instances_ppg_list = tolist(var.limit_instances_ppg) - limit_l4 = join(",", local.limit_l4_list) - limit_l4_list = tolist(var.limit_l4) - limit_l4_ppg = join(",", local.limit_l4_ppg_list) - limit_l4_ppg_list = tolist(var.limit_l4_ppg) - limit_l7 = join(",", local.limit_l7_list) - limit_l7_list = tolist(var.limit_l7) - limit_l7_ppg = join(",", local.limit_l7_ppg_list) - limit_l7_ppg_list = tolist(var.limit_l7_ppg) - limit_subnets = join(",", local.limit_subnets_list) - limit_subnets_list = tolist(var.limit_subnets) - limit_vpc_peer = join(",", local.limit_vpc_peer_list) - limit_vpc_peer_list = tolist(var.limit_vpc_peer) } ################################################ @@ -130,14 +113,6 @@ module "cloud-function" { } environment_variables = { - LIMIT_INSTANCES = local.limit_instances - LIMIT_INSTANCES_PPG = local.limit_instances_ppg - LIMIT_L4 = local.limit_l4 - LIMIT_L4_PPG = local.limit_l4_ppg - LIMIT_L7 = local.limit_l7 - LIMIT_L7_PPG = local.limit_l7_ppg - LIMIT_SUBNETS = local.limit_subnets - LIMIT_VPC_PEER = local.limit_vpc_peer MONITORED_PROJECTS_LIST = local.projects MONITORING_PROJECT_ID = module.project-monitoring.project_id ORGANIZATION_ID = var.organization_id diff --git a/examples/cloud-operations/network-dashboard/variables.tf b/examples/cloud-operations/network-dashboard/variables.tf index 7170a513..7e4237ca 100644 --- a/examples/cloud-operations/network-dashboard/variables.tf +++ b/examples/cloud-operations/network-dashboard/variables.tf @@ -75,69 +75,4 @@ variable "region" { variable "zone" { description = "Zone used to deploy vms" default = "europe-west1-b" -} - -variable "limit_l4" { - description = "Maximum number of forwarding rules for Internal TCP/UDP Load Balancing per network." - type = list(string) - default = [ - "default_value", "75", - ] -} - -variable "limit_l7" { - description = "Maximum number of forwarding rules for Internal HTTP(S) Load Balancing per network." - type = list(string) - default = [ - "default_value", "75", - ] -} - -variable "limit_subnets" { - description = "Maximum number of subnet IP ranges (primary and secondary) per peering group" - type = list(string) - default = [ - "default_value", "400", - ] -} - -variable "limit_instances" { - description = "Maximum number of instances per network" - type = list(string) - default = [ - "default_value", "15000", - ] -} - -variable "limit_instances_ppg" { - description = "Maximum number of instances per peering group." - type = list(string) - default = [ - "default_value", "15000", - ] -} - -variable "limit_vpc_peer" { - description = "Maximum number of peering VPC peerings per network." - type = list(string) - default = [ - "default_value", "25", - "test-vpc", "40", - ] -} - -variable "limit_l4_ppg" { - description = "Maximum number of forwarding rules for Internal TCP/UDP Load Balancing per network." - type = list(string) - default = [ - "default_value", "175", - ] -} - -variable "limit_l7_ppg" { - description = "Maximum number of forwarding rules for Internal HTTP(S) Load Balancing per network." - type = list(string) - default = [ - "default_value", "175", - ] } \ No newline at end of file From 0344e7df8cb4e7ea4326a33f7eb19d466d31f90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Mon, 28 Mar 2022 18:47:11 +0200 Subject: [PATCH 17/23] formatting --- .../network-dashboard/cloud-function/main.py | 83 ++++++++++++------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 6f7e2135..8875ff8d 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -59,28 +59,36 @@ def main(event, context): subnet_range_dict = get_subnet_ranges_dict() # Per Network metrics - get_gce_instances_data(metrics_dict, gce_instance_dict, limits_dict['number_of_instances_limit']) - get_l4_forwarding_rules_data(metrics_dict, l4_forwarding_rules_dict, limits_dict['internal_forwarding_rules_l4_limit']) - get_vpc_peering_data(metrics_dict, limits_dict['number_of_vpc_peerings_limit']) + get_gce_instances_data(metrics_dict, gce_instance_dict, + limits_dict['number_of_instances_limit']) + get_l4_forwarding_rules_data( + metrics_dict, l4_forwarding_rules_dict, + limits_dict['internal_forwarding_rules_l4_limit']) + get_vpc_peering_data(metrics_dict, + limits_dict['number_of_vpc_peerings_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], - gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC, limits_dict['number_of_instances_ppg_limit']) + gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC, + limits_dict['number_of_instances_ppg_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"] ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict, - L4_FORWARDING_RULES_LIMIT_METRIC, limits_dict['internal_forwarding_rules_l4_ppg_limit']) + L4_FORWARDING_RULES_LIMIT_METRIC, + limits_dict['internal_forwarding_rules_l4_ppg_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"] ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict, - L7_FORWARDING_RULES_LIMIT_METRIC, limits_dict['internal_forwarding_rules_l7_ppg_limit']) + L7_FORWARDING_RULES_LIMIT_METRIC, + limits_dict['internal_forwarding_rules_l7_ppg_limit']) get_pgg_data( metrics_dict["metrics_per_peering_group"] ["subnet_ranges_per_peering_group"], subnet_range_dict, - SUBNET_RANGES_LIMIT_METRIC, limits_dict['number_of_subnet_IP_ranges_limit']) + SUBNET_RANGES_LIMIT_METRIC, + limits_dict['number_of_subnet_IP_ranges_limit']) return 'Function executed successfully' @@ -300,7 +308,7 @@ def create_metrics(): for network_link, limit_value in sub_metric["values"].items(): limits_dict_for_metric[network_link] = limit_value limits_dict[sub_metric["name"]] = limits_dict_for_metric - + return metrics_dict, limits_dict except yaml.YAMLError as exc: print(exc) @@ -383,7 +391,8 @@ def get_vpc_peering_data(metrics_dict, limit_dict): None ''' for project in MONITORED_PROJECTS_LIST: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data(project, limit_dict) + active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data( + project, limit_dict) for peering in active_vpc_peerings: write_data_to_metric( project, peering['active_peerings'], @@ -442,7 +451,7 @@ def gather_vpc_peerings_data(project_id, limit_dict): else: peerings_count = 0 active_peerings_count = 0 - + network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network['name']}" network_limit = get_limit_ppg(network_link, limit_dict) @@ -484,7 +493,8 @@ def get_limit_ppg(network_link, limit_dict): return 0 -def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict, limit_dict): +def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict, + limit_dict): ''' Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. @@ -498,8 +508,8 @@ def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict, limit_dict for project in MONITORED_PROJECTS_LIST: network_dict = get_networks(project) - current_quota_limit = get_quota_current_limit(f"projects/{project}", - L4_FORWARDING_RULES_LIMIT_METRIC) + current_quota_limit = get_quota_current_limit( + f"projects/{project}", L4_FORWARDING_RULES_LIMIT_METRIC) current_quota_limit_view = customize_quota_view(current_quota_limit) @@ -556,8 +566,8 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_dict): current_quota_limit = get_quota_current_limit(f"projects/{project}", limit_metric) current_quota_limit_view = customize_quota_view(current_quota_limit) - limit = get_limit_network(network_dict, network_link, current_quota_limit_view, - limit_dict) + limit = get_limit_network(network_dict, network_link, + current_quota_limit_view, limit_dict) usage = 0 if network_link in usage_dict: @@ -575,11 +585,12 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_dict): peered_usage = usage_dict[peered_network_link] peering_project_limit = customize_quota_view( - get_quota_current_limit(f"projects/{peered_network_dict['project_id']}", - limit_metric)) + get_quota_current_limit( + f"projects/{peered_network_dict['project_id']}", limit_metric)) - peered_limit = get_limit_network(peered_network_dict, peered_network_link, peering_project_limit, - limit_dict) + peered_limit = get_limit_network(peered_network_dict, + peered_network_link, + peering_project_limit, limit_dict) # Here we add usage and limit to the peered network dictionary peered_network_dict["usage"] = peered_usage peered_network_dict["limit"] = peered_limit @@ -621,7 +632,8 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) - limit_step1 = max(network_dict['limit'], get_limit_ppg(network_link, limit_dict)) + limit_step1 = max(network_dict['limit'], + get_limit_ppg(network_link, limit_dict)) # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network limit_step2 = [] @@ -646,6 +658,7 @@ def count_effective_limit(project_id, network_dict, usage_metric_name, write_data_to_metric(project_id, utilization, utilization_metric_name, network_dict['network_name']) + def get_networks(project_id): ''' Returns a dictionary of all networks in a project. @@ -662,10 +675,15 @@ def get_networks(project_id): for network in response['items']: network_name = network['name'] network_id = network['id'] - d = {'project_id': project_id, 'network_name': network_name, 'network_id': network_id} + d = { + 'project_id': project_id, + 'network_name': network_name, + 'network_id': network_id + } network_dict.append(d) return network_dict + # TODO: list all routers (https://cloud.google.com/compute/docs/reference/rest/v1/routers/list) then https://cloud.google.com/compute/docs/reference/rest/v1/routers/getRouterStatus def get_routes(project_id): ''' @@ -683,10 +701,15 @@ def get_routes(project_id): for router in response['items']: network_name = router['name'] network_id = router['id'] - d = {'project_id': project_id, 'network name': network_name, 'network id': network_id} + d = { + 'project_id': project_id, + 'network name': network_name, + 'network id': network_id + } network_dict.append(d) return network_dict + def gather_peering_data(project_id): ''' Returns a dictionary of all peerings for all networks in a project. @@ -758,6 +781,7 @@ def get_network_id(project_id, network_name): return network_id + def get_quota_current_limit(project_link, metric_name): ''' Retrieves limit for a specific metric. @@ -814,11 +838,10 @@ def set_limits(network_dict, quota_limit, limit_dict): ''' network_dict['limit'] = None - + if quota_limit: for net in quota_limit: - if net['network_id'] == network_dict[ - 'network_id']: + if net['network_id'] == network_dict['network_id']: network_dict['limit'] = net['value'] return @@ -833,6 +856,7 @@ def set_limits(network_dict, quota_limit, limit_dict): print(f"Error: Couldn't find limit for {network_link}") network_dict['limit'] = 0 + def get_limit_network(network_dict, network_link, quota_limit, limit_dict): ''' Returns limit for a specific network and metric, using the GCP quota metrics or the values in the yaml file if not found. @@ -847,9 +871,9 @@ def get_limit_network(network_dict, network_link, quota_limit, limit_dict): ''' if quota_limit: for net in quota_limit: - if net['network_id'] == network_dict['network_id']: + if net['network_id'] == network_dict['network_id']: return net['value'] - + if network_link in limit_dict: return limit_dict[network_link] else: @@ -902,6 +926,7 @@ def write_data_to_metric(monitored_project_id, value, metric_name, # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error # Implement exponential backoff retries? try: - client.create_time_series(name=MONITORING_PROJECT_LINK, time_series=[series]) + client.create_time_series(name=MONITORING_PROJECT_LINK, + time_series=[series]) except Exception as e: - print(e) \ No newline at end of file + print(e) From 1e60249c77be4813864611deac88795a973f3547 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:08:49 +0200 Subject: [PATCH 18/23] Bump minimist in /examples/serverless/api-gateway/function (#600) Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../api-gateway/function/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/serverless/api-gateway/function/package-lock.json b/examples/serverless/api-gateway/function/package-lock.json index 6bf7ab5c..da027c38 100644 --- a/examples/serverless/api-gateway/function/package-lock.json +++ b/examples/serverless/api-gateway/function/package-lock.json @@ -896,9 +896,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/ms": { "version": "2.0.0", @@ -2125,9 +2125,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "ms": { "version": "2.0.0", From 1bc9929c813c68c09a35123cb5837f5baf91dc18 Mon Sep 17 00:00:00 2001 From: Elia <79325566+eliamaldini@users.noreply.github.com> Date: Wed, 30 Mar 2022 10:20:10 +0200 Subject: [PATCH 19/23] Update vpn-spoke-prod.tf (#602) Fixed region ew4 --- fast/stages/02-networking-vpn/vpn-spoke-prod.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fast/stages/02-networking-vpn/vpn-spoke-prod.tf b/fast/stages/02-networking-vpn/vpn-spoke-prod.tf index ff635194..3001c4ec 100644 --- a/fast/stages/02-networking-vpn/vpn-spoke-prod.tf +++ b/fast/stages/02-networking-vpn/vpn-spoke-prod.tf @@ -82,7 +82,7 @@ module "landing-to-prod-ew4-vpn" { source = "../../../modules/net-vpn-ha" project_id = module.landing-project.project_id network = module.landing-vpc.self_link - region = "europe-west1" + region = "europe-west4" name = "vpn-to-prod-ew4" router_create = true router_name = "landing-vpn-ew4" @@ -112,7 +112,7 @@ module "prod-to-landing-ew4-vpn" { source = "../../../modules/net-vpn-ha" project_id = module.prod-spoke-project.project_id network = module.prod-spoke-vpc.self_link - region = "europe-west1" + region = "europe-west4" name = "vpn-to-landing-ew4" router_create = true router_name = "prod-spoke-vpn-ew4" From db37e43a99c877cd97ee9d0e042cd91e6127859c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Wed, 30 Mar 2022 17:03:31 +0200 Subject: [PATCH 20/23] Adding Dynamic Routes per Network as a new metric. --- .../network-dashboard/README.md | 7 +- .../network-dashboard/cloud-function/main.py | 171 ++++++++++-- .../cloud-function/metrics.yaml | 12 + .../dashboards/quotas-utilization.json | 253 ++++++++++-------- .../network-dashboard/main.tf | 2 +- 5 files changed, 305 insertions(+), 140 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/README.md b/examples/cloud-operations/network-dashboard/README.md index 2f958375..e9e79338 100644 --- a/examples/cloud-operations/network-dashboard/README.md +++ b/examples/cloud-operations/network-dashboard/README.md @@ -41,13 +41,18 @@ The Cloud Function currently tracks usage, limit and utilization of: - internal forwarding rules for internal L7 load balancers per VPC - internal forwarding rules for internal L4 load balancers per VPC peering group - internal forwarding rules for internal L7 load balancers per VPC peering group +- Dynamic routes per VPC It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. +Note that metrics are created in the cloud-function/metrics.yaml file. + +You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`. + ## Next steps and ideas In a future release, we could support: - Static routes per VPC / per VPC peering group -- Dynamic routes per VPC / per VPC peering group +- Dynamic routes per VPC peering group - Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) - Subnet IP ranges utilization diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index 8875ff8d..d81e04b8 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -66,7 +66,10 @@ def main(event, context): limits_dict['internal_forwarding_rules_l4_limit']) get_vpc_peering_data(metrics_dict, limits_dict['number_of_vpc_peerings_limit']) + get_dynamic_routes(metrics_dict, + limits_dict['dynamic_routes_per_network_limit']) + # Per VPC peering group metrics get_pgg_data( metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], gce_instance_dict, GCE_INSTANCES_LIMIT_METRIC, @@ -88,7 +91,7 @@ def main(event, context): metrics_dict["metrics_per_peering_group"] ["subnet_ranges_per_peering_group"], subnet_range_dict, SUBNET_RANGES_LIMIT_METRIC, - limits_dict['number_of_subnet_IP_ranges_limit']) + limits_dict['number_of_subnet_IP_ranges_ppg_limit']) return 'Function executed successfully' @@ -361,11 +364,9 @@ def get_gce_instances_data(metrics_dict, gce_instance_dict, limit_dict): for net in network_dict: set_limits(net, current_quota_limit_view, limit_dict) - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network_name']}" - usage = 0 - if network_link in gce_instance_dict: - usage = gce_instance_dict[network_link] + if net['self_link'] in gce_instance_dict: + usage = gce_instance_dict[net['self_link']] write_data_to_metric( project, usage, metrics_dict["metrics_per_network"] @@ -516,11 +517,9 @@ def get_l4_forwarding_rules_data(metrics_dict, forwarding_rules_dict, for net in network_dict: set_limits(net, current_quota_limit_view, limit_dict) - network_link = f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{net['network_name']}" - usage = 0 - if network_link in forwarding_rules_dict: - usage = forwarding_rules_dict[network_link] + if net['self_link'] in forwarding_rules_dict: + usage = forwarding_rules_dict[net['self_link']] write_data_to_metric( project, usage, metrics_dict["metrics_per_network"] @@ -599,7 +598,7 @@ def get_pgg_data(metric_dict, usage_dict, limit_metric, limit_dict): metric_dict["limit"]["name"], metric_dict["utilization"]["name"], limit_dict) print( - f"Wrote {metric_dict['usage']['name']} to metric for peering group {network_dict['network_name']} in {project}" + f"Wrote {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project}" ) @@ -675,39 +674,153 @@ def get_networks(project_id): for network in response['items']: network_name = network['name'] network_id = network['id'] + self_link = network['selfLink'] d = { 'project_id': project_id, 'network_name': network_name, - 'network_id': network_id + 'network_id': network_id, + 'self_link': self_link } network_dict.append(d) return network_dict -# TODO: list all routers (https://cloud.google.com/compute/docs/reference/rest/v1/routers/list) then https://cloud.google.com/compute/docs/reference/rest/v1/routers/getRouterStatus -def get_routes(project_id): +def get_dynamic_routes(metrics_dict, limits_dict): ''' - Returns a dictionary of all dynamic routes in a project. + Writes all dynamic routes per VPC to custom metrics. Parameters: - project_id (string): Project ID for the project containing the networks. + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. Returns: - network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) + network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s). ''' - request = service.routers().list(project=project_id) + routers_dict = get_routers() + + for project_id in MONITORED_PROJECTS_LIST: + network_dict = get_networks(project_id) + + for network in network_dict: + sum_routes = get_routes_for_network(network['self_link'], project_id, + routers_dict) + + if network['self_link'] in limits_dict: + limit = limits_dict[network['self_link']] + else: + if 'default_value' in limits_dict: + limit = limits_dict['default_value'] + else: + print("Error: couldn't find limit for dynamic routes.") + break + + utilization = sum_routes / limit + + write_data_to_metric( + project_id, sum_routes, metrics_dict["metrics_per_network"] + ["dynamic_routes_per_network"]["usage"]["name"], + network['network_name']) + write_data_to_metric( + project_id, limit, metrics_dict["metrics_per_network"] + ["dynamic_routes_per_network"]["limit"]["name"], + network['network_name']) + write_data_to_metric( + project_id, utilization, metrics_dict["metrics_per_network"] + ["dynamic_routes_per_network"]["utilization"]["name"], + network['network_name']) + + print("Wrote metrics for dynamic routes for VPCs in project", project_id) + + +def get_routes_for_network(network_link, project_id, routers_dict): + ''' + Returns a the number of dynamic routes for a given network + + Parameters: + network_link (string): Network self link. + project_id (string): Project ID containing the network. + routers_dict (dictionary of string: list of string): Dictionary with key as network link and value as list of router links. + Returns: + sum_routes (int): Number of routes in that network. + ''' + sum_routes = 0 + + if network_link in routers_dict: + for router_link in routers_dict[network_link]: + # Router link is using the following format: + # 'https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/routers/ROUTER_NAME' + start = router_link.find("/regions/") + len("/regions/") + end = router_link.find("/routers/") + router_region = router_link[start:end] + router_name = router_link.split('/routers/')[1] + routes = get_routes_for_router(project_id, router_region, router_name) + + sum_routes += routes + + return sum_routes + + +def get_routes_for_router(project_id, router_region, router_name): + ''' + Returns the same of dynamic routes learned by a specific Cloud Router instance + + Parameters: + project_id (string): Project ID for the project containing the Cloud Router. + router_region (string): GCP region for the Cloud Router. + router_name (string): Cloud Router name. + Returns: + sum_routes (int): Number of dynamic routes learned by the Cloud Router. + ''' + request = service.routers().getRouterStatus(project=project_id, + region=router_region, + router=router_name) response = request.execute() - network_dict = [] - if 'items' in response: - for router in response['items']: - network_name = router['name'] - network_id = router['id'] - d = { - 'project_id': project_id, - 'network name': network_name, - 'network id': network_id - } - network_dict.append(d) - return network_dict + + sum_routes = 0 + + if 'result' in response: + for peer in response['result']['bgpPeerStatus']: + sum_routes += peer['numLearnedRoutes'] + + return sum_routes + + +def get_routers(): + ''' + Returns a dictionary of all Cloud Routers in the GCP organization. + + Parameters: + None + Returns: + routers_dict (dictionary of string: list of string): Key is the network link and value is a list of router links. + ''' + client = asset_v1.AssetServiceClient() + + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + routers_dict = {} + + response = client.search_all_resources( + request={ + "scope": f"organizations/{ORGANIZATION_ID}", + "asset_types": ["compute.googleapis.com/Router"], + "read_mask": read_mask, + }) + for resource in response: + network_link = None + router_link = None + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "network": + network_link = field_value + if field_name == "selfLink": + router_link = field_value + + if network_link in routers_dict: + routers_dict[network_link].append(router_link) + else: + routers_dict[network_link] = [router_link] + + return routers_dict def gather_peering_data(project_id): diff --git a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml index a9772b5c..466293fb 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/examples/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -76,6 +76,18 @@ metrics_per_network: utilization: name: internal_forwarding_rules_l7_utilization description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per Vnetwork - utilization. + dynamic_routes_per_network: + usage: + name: dynamic_routes_per_network_usage + description: Number of Dynamic routes per network - usage. + limit: + name: dynamic_routes_per_network_limit + description: Number of Dynamic routes per network - limit. + values: + default_value: 100 + utilization: + name: dynamic_routes_per_network_utilization + description: Number of Dynamic routes per network - utilization. metrics_per_peering_group: l4_forwarding_rules_per_peering_group: usage: diff --git a/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 794d6123..e9059c3d 100644 --- a/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/examples/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -4,347 +4,382 @@ "columns": 12, "tiles": [ { + "width": 6, "height": 4, "widget": { "title": "internal_forwarding_rules_l4_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "1800s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6 + } }, { + "yPos": 12, + "width": 6, "height": 4, "widget": { "title": "internal_forwarding_rules_l7_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6, - "xPos": 6 + } }, { + "yPos": 8, + "width": 6, "height": 4, "widget": { "title": "number_of_instances_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6, - "yPos": 8 + } }, { + "xPos": 6, + "yPos": 4, + "width": 6, "height": 4, "widget": { "title": "number_of_vpc_peerings_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6, - "xPos": 6, - "yPos": 4 + } }, { + "yPos": 4, + "width": 6, "height": 4, "widget": { "title": "number_of_active_vpc_peerings_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_INTERPOLATE" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" - } - } - }, - "width": 6, - "yPos": 4 - }, - { - "height": 4, - "widget": { - "title": "number_of_subnet_IP_ranges_ppg_utilization", - "xyChart": { + }, "chartOptions": { "mode": "COLOR" - }, + } + } + } + }, + { + "yPos": 16, + "width": 6, + "height": 4, + "widget": { + "title": "subnet_IP_ranges_ppg_utilization", + "xyChart": { "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6, - "xPos": 6, - "yPos": 8 + } }, { + "xPos": 6, + "width": 6, "height": 4, "widget": { "title": "internal_forwarding_rules_l4_ppg_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6, - "yPos": 12 + } }, { + "xPos": 6, + "yPos": 12, + "width": 6, "height": 4, "widget": { "title": "internal_forwarding_rules_l7_ppg_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, - "width": 6, - "xPos": 6, - "yPos": 12 + } }, { + "xPos": 6, + "yPos": 8, + "width": 6, "height": 4, "widget": { "title": "number_of_instances_ppg_utilization", "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, "dataSets": [ { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", "timeSeriesQuery": { "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" - }, - "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s" } } - } + }, + "plotType": "LINE", + "minAlignmentPeriod": "3600s", + "targetAxis": "Y1" } ], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" } } - }, + } + }, + { + "xPos": 6, + "yPos": 16, "width": 6, - "yPos": 16 + "height": 4, + "widget": { + "title": "dynamic_routes_per_network_utilization", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN" + }, + "secondaryAggregation": { + "alignmentPeriod": "60s" + } + } + }, + "plotType": "LINE", + "minAlignmentPeriod": "60s", + "targetAxis": "Y1" + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + }, + "chartOptions": { + "mode": "COLOR" + } + } + } } ] } diff --git a/examples/cloud-operations/network-dashboard/main.tf b/examples/cloud-operations/network-dashboard/main.tf index 8fb7963c..6378c95d 100644 --- a/examples/cloud-operations/network-dashboard/main.tf +++ b/examples/cloud-operations/network-dashboard/main.tf @@ -134,4 +134,4 @@ module "cloud-function" { resource "google_monitoring_dashboard" "dashboard" { dashboard_json = file("${path.module}/dashboards/quotas-utilization.json") project = module.project-monitoring.project_id -} +} \ No newline at end of file From 63a4d81de7db2c9f9e7295aa173d84cfe7bddc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Wed, 30 Mar 2022 17:46:01 +0200 Subject: [PATCH 21/23] Documentation fix. --- .../cloud-operations/network-dashboard/cloud-function/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cloud-operations/network-dashboard/cloud-function/main.py b/examples/cloud-operations/network-dashboard/cloud-function/main.py index d81e04b8..fc41ddc1 100644 --- a/examples/cloud-operations/network-dashboard/cloud-function/main.py +++ b/examples/cloud-operations/network-dashboard/cloud-function/main.py @@ -684,15 +684,15 @@ def get_networks(project_id): network_dict.append(d) return network_dict - def get_dynamic_routes(metrics_dict, limits_dict): ''' Writes all dynamic routes per VPC to custom metrics. Parameters: metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. + limits_dict (dictionary of string: int): key is network link (or 'default_value') and value is the limit for that network Returns: - network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s). + None ''' routers_dict = get_routers() From 59bb09b03e0c3f2e68bc9cf971d46b891589e964 Mon Sep 17 00:00:00 2001 From: Daniel Strebel Date: Thu, 31 Mar 2022 08:47:34 +0200 Subject: [PATCH 22/23] Add billing_type for Apigee Organization Module --- modules/apigee-organization/README.md | 1 + modules/apigee-organization/main.tf | 1 + modules/apigee-organization/variables.tf | 6 +++++- tests/modules/apigee_organization/fixture/main.tf | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/apigee-organization/README.md b/modules/apigee-organization/README.md index 36878896..11dda36f 100644 --- a/modules/apigee-organization/README.md +++ b/modules/apigee-organization/README.md @@ -110,6 +110,7 @@ module "apigee-organization" { | [apigee_envgroups](variables.tf#L22) | Apigee Environment Groups. | map(object({…})) | | {} | | [apigee_environments](variables.tf#L31) | Apigee Environment Names. | list(string) | | [] | | [authorized_network](variables.tf#L37) | VPC network self link (requires service network peering enabled (Used in Apigee X only). | string | | null | +| [billing_type](variables.tf#L75) | Billing type of the Apigee organization. | string | | null | | [database_encryption_key](variables.tf#L43) | Cloud KMS key self link (e.g. `projects/foo/locations/us/keyRings/bar/cryptoKeys/baz`) used for encrypting the data that is stored and replicated across runtime instances (immutable, used in Apigee X only). | string | | null | | [description](variables.tf#L49) | Description of the Apigee Organization. | string | | "Apigee Organization created by tf module" | | [display_name](variables.tf#L55) | Display Name of the Apigee Organization. | string | | null | diff --git a/modules/apigee-organization/main.tf b/modules/apigee-organization/main.tf index fe798f92..148711a9 100644 --- a/modules/apigee-organization/main.tf +++ b/modules/apigee-organization/main.tf @@ -31,6 +31,7 @@ resource "google_apigee_organization" "apigee_org" { display_name = var.display_name description = var.description runtime_type = var.runtime_type + billing_type = var.billing_type authorized_network = var.authorized_network runtime_database_encryption_key_name = var.database_encryption_key } diff --git a/modules/apigee-organization/variables.tf b/modules/apigee-organization/variables.tf index d7ab70da..b2b3eac9 100644 --- a/modules/apigee-organization/variables.tf +++ b/modules/apigee-organization/variables.tf @@ -72,4 +72,8 @@ variable "runtime_type" { } } - +variable "billing_type" { + description = "Billing type of the Apigee organization." + type = string + default = null +} diff --git a/tests/modules/apigee_organization/fixture/main.tf b/tests/modules/apigee_organization/fixture/main.tf index 49ad78b1..9dfb49bc 100644 --- a/tests/modules/apigee_organization/fixture/main.tf +++ b/tests/modules/apigee_organization/fixture/main.tf @@ -19,6 +19,7 @@ module "test" { project_id = "my-project" analytics_region = var.analytics_region runtime_type = "CLOUD" + billing_type = "EVALUATION" authorized_network = var.network apigee_environments = [ "eval1", From 42fa28b0753cd69a50999751fa064298080a7c67 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Thu, 31 Mar 2022 10:36:47 +0200 Subject: [PATCH 23/23] Fix data catalog policy tag output. --- modules/data-catalog-policy-tag/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data-catalog-policy-tag/outputs.tf b/modules/data-catalog-policy-tag/outputs.tf index fcd5cc29..1f0bb242 100644 --- a/modules/data-catalog-policy-tag/outputs.tf +++ b/modules/data-catalog-policy-tag/outputs.tf @@ -16,7 +16,7 @@ output "tags" { description = "Policy Tags." - value = { for k, v in google_data_catalog_policy_tag.default : v.id => v.name } + value = { for k, v in google_data_catalog_policy_tag.default : k => v.id } } output "taxonomy_id" {