Add network tag. Add KMS support in the DAG example.

examples/data-solutions/data-platform-foundations/03-composer.tf

@@ -67,6 +67,7 @@ resource "google_composer_environment" "orch-cmp-0" {
       env_variables = merge(
         var.composer_config.env_variables, {
           BQ_LOCATION        = var.location
+          DF_KMS_KEY         = try(var.service_encryption_keys.dataflow, null)
           DTL_L0_PRJ         = module.lake-0-project.project_id
           DTL_L0_BQ_DATASET  = module.lake-0-bq-0.dataset_id
           DTL_L0_GCS         = module.lake-0-cs-0.url

examples/data-solutions/data-platform-foundations/demo/datapipeline.py

@@ -61,8 +61,9 @@ TRF_NET_VPC = os.environ.get("TRF_NET_VPC")
 TRF_NET_SUBNET = os.environ.get("TRF_NET_SUBNET")
 TRF_SA_DF = os.environ.get("TRF_SA_DF")
 TRF_SA_BQ = os.environ.get("TRF_SA_BQ")
-DF_ZONE = os.environ.get("GCP_REGION") + "-b"
+DF_KMS_KEY = os.environ.get("DF_KMS_KEY", "")
 DF_REGION = os.environ.get("GCP_REGION")
+DF_ZONE = os.environ.get("GCP_REGION") + "-b"
 
 # --------------------------------------------------------------------------------
 # Set default arguments
@@ -90,7 +91,8 @@ default_args = {
     'tempLocation': LOD_GCS_STAGING + "/tmp",
     'serviceAccountEmail': LOD_SA_DF,
     'subnetwork': LOD_NET_SUBNET,
-    'ipConfiguration': "WORKER_IP_PRIVATE"
+    'ipConfiguration': "WORKER_IP_PRIVATE",
+    'kmsKeyName' : DF_KMS_KEY
   },
 }
 

fast/stages/02-networking-nva/data/firewall-rules/dev/rules.yaml

@@ -0,0 +1,35 @@
+# skip boilerplate check
+
+ingress-allow-composer-nodes:
+  description: "Allow traffic on Cloud Composer subnet"
+  direction: INGRESS
+  action: allow
+  sources: []
+  ranges:
+    - 10.128.48.0/24
+  targets:
+    - composer-worker
+  use_service_accounts: false
+  rules:
+    - protocol: tcp
+      ports:
+        - 80
+        - 443
+        - 3306
+        - 3307
+
+ingress-allow-dataflow-load:
+  description: "Allow traffic on Cloud Dataflow subnet"
+  direction: INGRESS
+  action: allow
+  sources: []
+  ranges: 
+    - 10.128.48.0/24  
+  targets: 
+    - dataflow
+  use_service_accounts: false
+  rules:
+    - protocol: tcp
+      ports:
+        - 12345
+        - 12346

fast/stages/02-networking-vpn/README.md

@@ -289,6 +289,7 @@ DNS configurations are centralised in the `dns.tf` file. Spokes delegate DNS res
 
 | name | description | modules | resources |
 |---|---|---|---|
+| [02-networking-providers.tf](./02-networking-providers.tf) | None |  |  |
 | [dns-dev.tf](./dns-dev.tf) | Development spoke DNS zones and peerings setup. | <code>dns</code> |  |
 | [dns-landing.tf](./dns-landing.tf) | Landing DNS zones and peerings setup. | <code>dns</code> |  |
 | [dns-prod.tf](./dns-prod.tf) | Production spoke DNS zones and peerings setup. | <code>dns</code> |  |

fast/stages/02-networking-vpn/data/firewall-rules/dev/rules.yaml

@@ -1,7 +1,7 @@
 # skip boilerplate check
 
 ingress-allow-composer-nodes:
-  description: "Allow traffic on Cloud Dataflow subnet"
+  description: "Allow traffic on Cloud Composer subnet"
   direction: INGRESS
   action: allow
   sources: []
@@ -23,9 +23,10 @@ ingress-allow-dataflow-load:
   direction: INGRESS
   action: allow
   sources: []
-  ranges:
-    - 10.128.48.0/24
-  targets: []
+  ranges: 
+    - 10.128.48.0/24  
+  targets: 
+    - dataflow
   use_service_accounts: false
   rules:
     - protocol: tcp

fast/stages/03-data-platform/dev/README.md

@@ -115,6 +115,7 @@ terraform apply
 
 | name | description | modules |
 |---|---|---|
+| [03-data-platform-dev-providers.tf](./03-data-platform-dev-providers.tf) | None |  |
 | [main.tf](./main.tf) | Data Platformy. | <code>data-platform-foundations</code> |
 | [outputs.tf](./outputs.tf) | Output variables. |  |
 | [variables.tf](./variables.tf) | Terraform Variables. |  |