302 lines
10 KiB
Python
302 lines
10 KiB
Python
# Copyright 2022 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
'''Compute resources discovery from Cloud Asset Inventory.
|
|
|
|
This plugin handles discovery for Compute resources via a broad org-level
|
|
scoped CAI search. Common resource attributes are parsed by a generic handler
|
|
function, which then delegates parsing of resource-level attributes to smaller
|
|
specialized functions, one per resource type.
|
|
'''
|
|
|
|
import logging
|
|
|
|
from . import HTTPRequest, Level, Resource, register_init, register_discovery
|
|
from .utils import parse_cai_results
|
|
|
|
CAI_URL = ('https://content-cloudasset.googleapis.com/v1'
|
|
'/{root}/assets'
|
|
'?contentType=RESOURCE&{asset_types}&pageSize=500')
|
|
LOGGER = logging.getLogger('net-dash.discovery.cai-compute')
|
|
TYPES = {
|
|
'addresses': 'compute.googleapis.com/Address',
|
|
'global_addresses': 'compute.googleapis.com/GlobalAddress',
|
|
'firewall_policies': 'compute.googleapis.com/FirewallPolicy',
|
|
'firewall_rules': 'compute.googleapis.com/Firewall',
|
|
'forwarding_rules': 'compute.googleapis.com/ForwardingRule',
|
|
'instances': 'compute.googleapis.com/Instance',
|
|
'networks': 'compute.googleapis.com/Network',
|
|
'subnetworks': 'compute.googleapis.com/Subnetwork',
|
|
'routers': 'compute.googleapis.com/Router',
|
|
'routes': 'compute.googleapis.com/Route',
|
|
'sql_instances': 'sqladmin.googleapis.com/Instance',
|
|
'filestore_instances': 'file.googleapis.com/Instance',
|
|
'memorystore_instances': 'redis.googleapis.com/Instance',
|
|
}
|
|
NAMES = {v: k for k, v in TYPES.items()}
|
|
|
|
|
|
def _get_parent(parent, resources):
|
|
'Extracts and returns resource parent and type.'
|
|
parent_type, parent_id = parent.split('/')[-2:]
|
|
if parent_type == 'projects':
|
|
project = resources['projects:number'].get(parent_id)
|
|
if project:
|
|
return {'project_id': project['project_id'], 'project_number': parent_id}
|
|
if parent_type == 'folders':
|
|
if parent_id in resources['folders']:
|
|
return {'parent': f'{parent_type}/{parent_id}'}
|
|
if resources.get('organization') == parent_id:
|
|
return {'parent': f'{parent_type}/{parent_id}'}
|
|
|
|
|
|
def _handle_discovery(resources, response, data):
|
|
'Processes the asset API response and returns parsed resources or next URL.'
|
|
LOGGER.info('discovery handle request')
|
|
for result in parse_cai_results(data, 'cai-compute', method='list'):
|
|
resource = _handle_resource(resources, result['assetType'],
|
|
result['resource'])
|
|
if not resource:
|
|
continue
|
|
yield resource
|
|
page_token = data.get('nextPageToken')
|
|
if page_token:
|
|
LOGGER.info('requesting next page')
|
|
url = _url(resources)
|
|
yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None)
|
|
|
|
|
|
def _handle_resource(resources, asset_type, data):
|
|
'Parses and returns a single resource. Calls resource-level handler.'
|
|
# general attributes shared by all resource types
|
|
attrs = data['data']
|
|
# we use the asset type as the discovery name sometimes does not match
|
|
# e.g. assetType = GlobalAddress but discoveryName = Address
|
|
resource_name = NAMES[asset_type]
|
|
resource = {
|
|
'id':
|
|
attrs.get('id'),
|
|
'name':
|
|
attrs['name'],
|
|
# Some resources (ex: Filestore) don't have a self_link, using parent + name in that case
|
|
'self_link':
|
|
f'{data["parent"]}/{attrs["name"]}'
|
|
if not 'selfLink' in attrs else _self_link(attrs['selfLink']),
|
|
'assetType':
|
|
asset_type
|
|
}
|
|
# derive parent type and id and skip if parent is not within scope
|
|
parent_data = _get_parent(data['parent'], resources)
|
|
if not parent_data:
|
|
LOGGER.debug(f'{resource["self_link"]} outside perimeter')
|
|
LOGGER.debug([
|
|
resources['organization'], resources['folders'],
|
|
resources['projects:number']
|
|
])
|
|
return
|
|
resource.update(parent_data)
|
|
# gets and calls the resource-level handler for type specific attributes
|
|
func = globals().get(f'_handle_{resource_name}')
|
|
if not callable(func):
|
|
raise SystemExit(f'specialized function missing for {resource_name}')
|
|
extra_attrs = func(resource, attrs)
|
|
if not extra_attrs:
|
|
return
|
|
resource.update(extra_attrs)
|
|
return Resource(resource_name, resource['self_link'], resource)
|
|
|
|
|
|
def _handle_addresses(resource, data):
|
|
'Handles address type resource data.'
|
|
network = data.get('network')
|
|
subnet = data.get('subnetwork')
|
|
return {
|
|
'address': data['address'],
|
|
'internal': data.get('addressType') == 'INTERNAL',
|
|
'purpose': data.get('purpose', ''),
|
|
'status': data.get('status', ''),
|
|
'network': None if not network else _self_link(network),
|
|
'subnetwork': None if not subnet else _self_link(subnet)
|
|
}
|
|
|
|
|
|
def _handle_firewall_policies(resource, data):
|
|
'Handles firewall policy type resource data.'
|
|
return {
|
|
'num_rules': len(data.get('rules', [])),
|
|
'num_tuples': data.get('ruleTupleCount', 0)
|
|
}
|
|
|
|
|
|
def _handle_firewall_rules(resource, data):
|
|
'Handles firewall type resource data.'
|
|
return {'network': _self_link(data['network'])}
|
|
|
|
|
|
def _handle_forwarding_rules(resource, data):
|
|
'Handles forwarding_rules type resource data.'
|
|
network = data.get('network')
|
|
region = data.get('region')
|
|
subnet = data.get('subnetwork')
|
|
return {
|
|
'address': data.get('IPAddress'),
|
|
'load_balancing_scheme': data.get('loadBalancingScheme', ''),
|
|
'network': None if not network else _self_link(network),
|
|
'psc_accepted': data.get('pscConnectionStatus') == 'ACCEPTED',
|
|
'region': None if not region else region.split('/')[-1],
|
|
'subnetwork': None if not subnet else _self_link(subnet)
|
|
}
|
|
|
|
|
|
def _handle_global_addresses(resource, data):
|
|
'Handles GlobalAddress type resource data (ex: PSA ranges).'
|
|
network = data.get('network')
|
|
return {
|
|
'address': data['address'],
|
|
'prefixLength': data.get('prefixLength') or None,
|
|
'internal': data.get('addressType') == 'INTERNAL',
|
|
'purpose': data.get('purpose', ''),
|
|
'status': data.get('status', ''),
|
|
'network': None if not network else _self_link(network),
|
|
}
|
|
|
|
|
|
def _handle_instances(resource, data):
|
|
'Handles instance type resource data.'
|
|
if data['status'] != 'RUNNING':
|
|
return
|
|
networks = [{
|
|
'network': _self_link(i['network']),
|
|
'subnetwork': _self_link(i['subnetwork'])
|
|
} for i in data.get('networkInterfaces', [])]
|
|
return {'zone': data['zone'], 'networks': networks}
|
|
|
|
|
|
def _handle_networks(resource, data):
|
|
'Handles network type resource data.'
|
|
peerings = [{
|
|
'active': p['state'] == 'ACTIVE',
|
|
'name': p['name'],
|
|
'network': _self_link(p['network']),
|
|
'project_id': _self_link(p['network']).split('/')[1]
|
|
} for p in data.get('peerings', [])]
|
|
subnets = [_self_link(s) for s in data.get('subnetworks', [])]
|
|
return {'peerings': peerings, 'subnetworks': subnets}
|
|
|
|
|
|
def _handle_routers(resource, data):
|
|
'Handles router type resource data.'
|
|
return {
|
|
'network': _self_link(data['network']),
|
|
'region': data['region'].split('/')[-1]
|
|
}
|
|
|
|
|
|
def _handle_routes(resource, data):
|
|
'Handles route type resource data.'
|
|
hop = [
|
|
a.removeprefix('nextHop').lower() for a in data if a.startswith('nextHop')
|
|
]
|
|
return {'next_hop_type': hop[0], 'network': _self_link(data['network'])}
|
|
|
|
|
|
def _handle_sql_instances(resource, data):
|
|
'Handles cloud sql instance type resource data.'
|
|
return {
|
|
'name': data['name'],
|
|
'self_link': _self_link(data['selfLink']),
|
|
'ipAddresses': [
|
|
i['ipAddress'] for i in data['ipAddresses'] if i['type'] == 'PRIVATE'
|
|
],
|
|
'region': data['region'],
|
|
'availabilityType': data['settings']['availabilityType'],
|
|
'network': data['settings']['ipConfiguration']['privateNetwork']
|
|
}
|
|
|
|
|
|
def _handle_filestore_instances(resource, data):
|
|
'Handles filestore instance type resource data.'
|
|
return {
|
|
# Getting only the instance name, removing the rest
|
|
'name': data['name'].split('/')[-1],
|
|
# Is a list but for now, only one network is supported for Filestore
|
|
'network': data['networks'][0]['network'],
|
|
'reservedIpRange': data['networks'][0]['reservedIpRange'],
|
|
'ipAddresses': data['networks'][0]['ipAddresses']
|
|
}
|
|
|
|
|
|
def _handle_memorystore_instances(resource, data):
|
|
'Handles Memorystore (Redis) instance type resource data.'
|
|
return {
|
|
# Getting only the instance name, removing the rest
|
|
'name':
|
|
data['name'].split('/')[-1],
|
|
'locationId':
|
|
data['locationId'],
|
|
'replicaCount':
|
|
0 if not 'replicaCount' in data else data['replicaCount'],
|
|
'network':
|
|
data['authorizedNetwork'],
|
|
'reservedIpRange':
|
|
'' if not 'reservedIpRange' in data else data['reservedIpRange'],
|
|
'host':
|
|
'' if not 'host' in data else data['host'],
|
|
}
|
|
|
|
|
|
def _handle_subnetworks(resource, data):
|
|
'Handles subnetwork type resource data.'
|
|
secondary_ranges = [{
|
|
'name': s['rangeName'],
|
|
'cidr_range': s['ipCidrRange']
|
|
} for s in data.get('secondaryIpRanges', [])]
|
|
return {
|
|
'cidr_range': data['ipCidrRange'],
|
|
'network': _self_link(data['network']),
|
|
'purpose': data.get('purpose'),
|
|
'region': data['region'].split('/')[-1],
|
|
'secondary_ranges': secondary_ranges
|
|
}
|
|
|
|
|
|
def _self_link(s):
|
|
'Removes initial part from self links.'
|
|
return '/'.join(s.split('/')[5:])
|
|
|
|
|
|
def _url(resources):
|
|
'Returns discovery URL'
|
|
discovery_root = resources['config:discovery_root']
|
|
asset_types = '&'.join(f'assetTypes={t}' for t in TYPES.values())
|
|
return CAI_URL.format(root=discovery_root, asset_types=asset_types)
|
|
|
|
|
|
@register_init
|
|
def init(resources):
|
|
'Prepares the datastructures for types managed here in the resource map.'
|
|
LOGGER.info('init')
|
|
for name in TYPES:
|
|
resources.setdefault(name, {})
|
|
|
|
|
|
@register_discovery(Level.PRIMARY, 10)
|
|
def start_discovery(resources, response=None, data=None):
|
|
'Plugin entry point, triggers discovery and handles requests and responses.'
|
|
LOGGER.info(f'discovery (has response: {response is not None})')
|
|
if response is None:
|
|
yield HTTPRequest(_url(resources), {}, None)
|
|
else:
|
|
for result in _handle_discovery(resources, response, data):
|
|
yield result
|