Add target_node and minor fixes (#389)
* Add separate BQ tables per asset type * Add missing BQ location * Add missing CF region * Change Scheduler job to non-generic * Add target_node, pass tests * Do tfdoc.py * Do terraform fmt * Do terraform fmt -recursive for tests' fixtures * Fix typo in test's variables
This commit is contained in:
parent
a4c48b8950
commit
5beba11058
|
@ -43,7 +43,7 @@ You can also create a dashboard connecting [Datalab](https://datastudio.google.c
|
||||||
|
|
||||||
| name | description | type | required | default |
|
| name | description | type | required | default |
|
||||||
|---|---|:---: |:---:|:---:|
|
|---|---|:---: |:---:|:---:|
|
||||||
| cai_config | Cloud Asset inventory export config. | <code title="object({ bq_dataset = string bq_table = string })">object({...})</code> | ✓ | |
|
| cai_config | Cloud Asset inventory export config. | <code title="object({ bq_dataset = string bq_table = string target_node = string })">object({...})</code> | ✓ | |
|
||||||
| project_id | Project id that references existing project. | <code title="">string</code> | ✓ | |
|
| project_id | Project id that references existing project. | <code title="">string</code> | ✓ | |
|
||||||
| *billing_account* | Billing account id used as default for new projects. | <code title="">string</code> | | <code title="">null</code> |
|
| *billing_account* | Billing account id used as default for new projects. | <code title="">string</code> | | <code title="">null</code> |
|
||||||
| *bundle_path* | Path used to write the intermediate Cloud Function code bundle. | <code title="">string</code> | | <code title="">./bundle.zip</code> |
|
| *bundle_path* | Path used to write the intermediate Cloud Function code bundle. | <code title="">string</code> | | <code title="">./bundle.zip</code> |
|
||||||
|
|
|
@ -50,17 +50,18 @@ def _configure_logging(verbose=True):
|
||||||
@click.option('--bq-project', required=True, help='Bigquery project to use.')
|
@click.option('--bq-project', required=True, help='Bigquery project to use.')
|
||||||
@click.option('--bq-dataset', required=True, help='Bigquery dataset to use.')
|
@click.option('--bq-dataset', required=True, help='Bigquery dataset to use.')
|
||||||
@click.option('--bq-table', required=True, help='Bigquery table name to use.')
|
@click.option('--bq-table', required=True, help='Bigquery table name to use.')
|
||||||
|
@click.option('--target-node', required=True, help='Node in Google Cloud resource hierarchy.')
|
||||||
@click.option('--read-time', required=False, help=(
|
@click.option('--read-time', required=False, help=(
|
||||||
'Day to take an asset snapshot in \'YYYYMMDD\' format, uses current day '
|
'Day to take an asset snapshot in \'YYYYMMDD\' format, uses current day '
|
||||||
' as default. Export will run at midnight of the specified day.'))
|
' as default. Export will run at midnight of the specified day.'))
|
||||||
@click.option('--verbose', is_flag=True, help='Verbose output')
|
@click.option('--verbose', is_flag=True, help='Verbose output')
|
||||||
def main_cli(project=None, bq_project=None, bq_dataset=None, bq_table=None,
|
def main_cli(project=None, bq_project=None, bq_dataset=None, bq_table=None, target_node=None,
|
||||||
read_time=None, verbose=False):
|
read_time=None, verbose=False):
|
||||||
'''Trigger Cloud Asset inventory export to Bigquery. Data will be stored in
|
'''Trigger Cloud Asset inventory export to Bigquery. Data will be stored in
|
||||||
the dataset specified on a dated table with the name specified.
|
the dataset specified on a dated table with the name specified.
|
||||||
'''
|
'''
|
||||||
try:
|
try:
|
||||||
_main(project, bq_project, bq_dataset, bq_table, read_time, verbose)
|
_main(project, bq_project, bq_dataset, bq_table, target_node, read_time, verbose)
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
logging.exception('exception raised')
|
logging.exception('exception raised')
|
||||||
|
|
||||||
|
@ -78,25 +79,25 @@ def main(event, context):
|
||||||
logging.exception('exception in cloud function entry point')
|
logging.exception('exception in cloud function entry point')
|
||||||
|
|
||||||
|
|
||||||
def _main(project=None, bq_project=None, bq_dataset=None, bq_table=None, read_time=None, verbose=False):
|
def _main(project=None, bq_project=None, bq_dataset=None, bq_table=None, target_node=None, read_time=None, verbose=False):
|
||||||
'Module entry point used by cli and cloud function wrappers.'
|
'Module entry point used by cli and cloud function wrappers.'
|
||||||
|
|
||||||
_configure_logging(verbose)
|
_configure_logging(verbose)
|
||||||
if not read_time:
|
if not read_time:
|
||||||
read_time = datetime.datetime.now()
|
read_time = datetime.datetime.now()
|
||||||
client = asset_v1.AssetServiceClient()
|
client = asset_v1.AssetServiceClient()
|
||||||
parent = 'projects/%s' % project
|
|
||||||
content_type = asset_v1.ContentType.RESOURCE
|
content_type = asset_v1.ContentType.RESOURCE
|
||||||
output_config = asset_v1.OutputConfig()
|
output_config = asset_v1.OutputConfig()
|
||||||
output_config.bigquery_destination.dataset = 'projects/%s/datasets/%s' % (
|
output_config.bigquery_destination.dataset = 'projects/%s/datasets/%s' % (
|
||||||
bq_project, bq_dataset)
|
bq_project, bq_dataset)
|
||||||
output_config.bigquery_destination.table = '%s_%s' % (
|
output_config.bigquery_destination.table = '%s_%s' % (
|
||||||
bq_table, read_time.strftime('%Y%m%d'))
|
bq_table, read_time.strftime('%Y%m%d'))
|
||||||
|
output_config.bigquery_destination.separate_tables_per_asset_type = True
|
||||||
output_config.bigquery_destination.force = True
|
output_config.bigquery_destination.force = True
|
||||||
try:
|
try:
|
||||||
response = client.export_assets(
|
response = client.export_assets(
|
||||||
request={
|
request={
|
||||||
'parent': parent,
|
'parent': target_node,
|
||||||
'read_time': read_time,
|
'read_time': read_time,
|
||||||
'content_type': content_type,
|
'content_type': content_type,
|
||||||
'output_config': output_config
|
'output_config': output_config
|
||||||
|
@ -105,7 +106,7 @@ def _main(project=None, bq_project=None, bq_dataset=None, bq_table=None, read_ti
|
||||||
except (GoogleAPIError, googleapiclient.errors.HttpError) as e:
|
except (GoogleAPIError, googleapiclient.errors.HttpError) as e:
|
||||||
logging.debug('API Error: %s', e, exc_info=True)
|
logging.debug('API Error: %s', e, exc_info=True)
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
'Error fetching Asset Inventory entries (project: %s)' % parent, e)
|
'Error fetching Asset Inventory entries (resource manager node: %s)' % target_node, e)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -73,6 +73,7 @@ module "pubsub" {
|
||||||
module "cf" {
|
module "cf" {
|
||||||
source = "../../modules/cloud-function"
|
source = "../../modules/cloud-function"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
|
region = var.region
|
||||||
name = var.name
|
name = var.name
|
||||||
bucket_name = "${var.name}-${random_pet.random.id}"
|
bucket_name = "${var.name}-${random_pet.random.id}"
|
||||||
bucket_config = {
|
bucket_config = {
|
||||||
|
@ -108,8 +109,8 @@ resource "google_app_engine_application" "app" {
|
||||||
resource "google_cloud_scheduler_job" "job" {
|
resource "google_cloud_scheduler_job" "job" {
|
||||||
project = google_app_engine_application.app.project
|
project = google_app_engine_application.app.project
|
||||||
region = var.region
|
region = var.region
|
||||||
name = "test-job"
|
name = "cai-export-job"
|
||||||
description = "test http job"
|
description = "CAI Export Job"
|
||||||
schedule = "* 9 * * 1"
|
schedule = "* 9 * * 1"
|
||||||
time_zone = "Etc/UTC"
|
time_zone = "Etc/UTC"
|
||||||
|
|
||||||
|
@ -117,10 +118,11 @@ resource "google_cloud_scheduler_job" "job" {
|
||||||
attributes = {}
|
attributes = {}
|
||||||
topic_name = module.pubsub.topic.id
|
topic_name = module.pubsub.topic.id
|
||||||
data = base64encode(jsonencode({
|
data = base64encode(jsonencode({
|
||||||
project = module.project.project_id
|
project = module.project.project_id
|
||||||
bq_project = module.project.project_id
|
bq_project = module.project.project_id
|
||||||
bq_dataset = var.cai_config.bq_dataset
|
bq_dataset = var.cai_config.bq_dataset
|
||||||
bq_table = var.cai_config.bq_table
|
bq_table = var.cai_config.bq_table
|
||||||
|
target_node = var.cai_config.target_node
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -133,6 +135,7 @@ module "bq" {
|
||||||
source = "../../modules/bigquery-dataset"
|
source = "../../modules/bigquery-dataset"
|
||||||
project_id = module.project.project_id
|
project_id = module.project.project_id
|
||||||
id = var.cai_config.bq_dataset
|
id = var.cai_config.bq_dataset
|
||||||
|
location = var.region
|
||||||
access = {
|
access = {
|
||||||
owner = { role = "OWNER", type = "user" }
|
owner = { role = "OWNER", type = "user" }
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,8 +29,9 @@ variable "bundle_path" {
|
||||||
variable "cai_config" {
|
variable "cai_config" {
|
||||||
description = "Cloud Asset inventory export config."
|
description = "Cloud Asset inventory export config."
|
||||||
type = object({
|
type = object({
|
||||||
bq_dataset = string
|
bq_dataset = string
|
||||||
bq_table = string
|
bq_table = string
|
||||||
|
target_node = string
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,12 +19,14 @@ variable "billing_account" {
|
||||||
|
|
||||||
variable "cai_config" {
|
variable "cai_config" {
|
||||||
type = object({
|
type = object({
|
||||||
bq_dataset = string
|
bq_dataset = string
|
||||||
bq_table = string
|
bq_table = string
|
||||||
|
target_node = string
|
||||||
})
|
})
|
||||||
default = {
|
default = {
|
||||||
bq_dataset = "my-dataset"
|
bq_dataset = "my-dataset"
|
||||||
bq_table = "my_table"
|
bq_table = "my_table"
|
||||||
|
target_node = "organization/1234567890"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue