From 55c78a132f223f15c738bb4802d0dd7d778358c8 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Fri, 18 Feb 2022 22:09:48 +0100 Subject: [PATCH] Add network tag. Add KMS support in the DAG example. --- .../data-platform-foundations/03-composer.tf | 1 + .../demo/datapipeline.py | 6 ++-- .../data/firewall-rules/dev/rules.yaml | 35 +++++++++++++++++++ fast/stages/02-networking-vpn/README.md | 1 + .../data/firewall-rules/dev/rules.yaml | 9 ++--- fast/stages/03-data-platform/dev/README.md | 1 + 6 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml diff --git a/examples/data-solutions/data-platform-foundations/03-composer.tf b/examples/data-solutions/data-platform-foundations/03-composer.tf index 6a23a6a1..3901fe78 100644 --- a/examples/data-solutions/data-platform-foundations/03-composer.tf +++ b/examples/data-solutions/data-platform-foundations/03-composer.tf @@ -67,6 +67,7 @@ resource "google_composer_environment" "orch-cmp-0" { env_variables = merge( var.composer_config.env_variables, { BQ_LOCATION = var.location + DF_KMS_KEY = try(var.service_encryption_keys.dataflow, null) DTL_L0_PRJ = module.lake-0-project.project_id DTL_L0_BQ_DATASET = module.lake-0-bq-0.dataset_id DTL_L0_GCS = module.lake-0-cs-0.url diff --git a/examples/data-solutions/data-platform-foundations/demo/datapipeline.py b/examples/data-solutions/data-platform-foundations/demo/datapipeline.py index 878d641a..fd633ebd 100644 --- a/examples/data-solutions/data-platform-foundations/demo/datapipeline.py +++ b/examples/data-solutions/data-platform-foundations/demo/datapipeline.py @@ -61,8 +61,9 @@ TRF_NET_VPC = os.environ.get("TRF_NET_VPC") TRF_NET_SUBNET = os.environ.get("TRF_NET_SUBNET") TRF_SA_DF = os.environ.get("TRF_SA_DF") TRF_SA_BQ = os.environ.get("TRF_SA_BQ") -DF_ZONE = os.environ.get("GCP_REGION") + "-b" +DF_KMS_KEY = os.environ.get("DF_KMS_KEY", "") DF_REGION = os.environ.get("GCP_REGION") +DF_ZONE = os.environ.get("GCP_REGION") + "-b" # -------------------------------------------------------------------------------- # Set default arguments @@ -90,7 +91,8 @@ default_args = { 'tempLocation': LOD_GCS_STAGING + "/tmp", 'serviceAccountEmail': LOD_SA_DF, 'subnetwork': LOD_NET_SUBNET, - 'ipConfiguration': "WORKER_IP_PRIVATE" + 'ipConfiguration': "WORKER_IP_PRIVATE", + 'kmsKeyName' : DF_KMS_KEY }, } diff --git a/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml b/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml new file mode 100644 index 00000000..cadf16ec --- /dev/null +++ b/fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml @@ -0,0 +1,35 @@ +# skip boilerplate check + +ingress-allow-composer-nodes: + description: "Allow traffic on Cloud Composer subnet" + direction: INGRESS + action: allow + sources: [] + ranges: + - 10.128.48.0/24 + targets: + - composer-worker + use_service_accounts: false + rules: + - protocol: tcp + ports: + - 80 + - 443 + - 3306 + - 3307 + +ingress-allow-dataflow-load: + description: "Allow traffic on Cloud Dataflow subnet" + direction: INGRESS + action: allow + sources: [] + ranges: + - 10.128.48.0/24 + targets: + - dataflow + use_service_accounts: false + rules: + - protocol: tcp + ports: + - 12345 + - 12346 diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md index b57f6b2d..86c9c3fe 100644 --- a/fast/stages/02-networking-vpn/README.md +++ b/fast/stages/02-networking-vpn/README.md @@ -289,6 +289,7 @@ DNS configurations are centralised in the `dns.tf` file. Spokes delegate DNS res | name | description | modules | resources | |---|---|---|---| +| [02-networking-providers.tf](./02-networking-providers.tf) | None | | | | [dns-dev.tf](./dns-dev.tf) | Development spoke DNS zones and peerings setup. | dns | | | [dns-landing.tf](./dns-landing.tf) | Landing DNS zones and peerings setup. | dns | | | [dns-prod.tf](./dns-prod.tf) | Production spoke DNS zones and peerings setup. | dns | | diff --git a/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml b/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml index dcc36d3e..cadf16ec 100644 --- a/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml +++ b/fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml @@ -1,7 +1,7 @@ # skip boilerplate check ingress-allow-composer-nodes: - description: "Allow traffic on Cloud Dataflow subnet" + description: "Allow traffic on Cloud Composer subnet" direction: INGRESS action: allow sources: [] @@ -23,9 +23,10 @@ ingress-allow-dataflow-load: direction: INGRESS action: allow sources: [] - ranges: - - 10.128.48.0/24 - targets: [] + ranges: + - 10.128.48.0/24 + targets: + - dataflow use_service_accounts: false rules: - protocol: tcp diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/03-data-platform/dev/README.md index 9c5c4259..313318c4 100644 --- a/fast/stages/03-data-platform/dev/README.md +++ b/fast/stages/03-data-platform/dev/README.md @@ -115,6 +115,7 @@ terraform apply | name | description | modules | |---|---|---| +| [03-data-platform-dev-providers.tf](./03-data-platform-dev-providers.tf) | None | | | [main.tf](./main.tf) | Data Platformy. | data-platform-foundations | | [outputs.tf](./outputs.tf) | Output variables. | | | [variables.tf](./variables.tf) | Terraform Variables. | |