diff --git a/blueprints/factories/bigquery-factory/README.md b/blueprints/factories/bigquery-factory/README.md new file mode 100644 index 00000000..01c098be --- /dev/null +++ b/blueprints/factories/bigquery-factory/README.md @@ -0,0 +1,88 @@ +# Google Cloud BQ Factory + +This module allows creation and management of BigQuery datasets and views as well as tables by defining them in well formatted `yaml` files. + +Yaml abstraction for BQ can simplify users onboarding and also makes creation of tables easier compared to HCL. + +Subfolders distinguish between views and tables and ensures easier navigation for users. + +This factory is based on the [BQ dataset module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/bigquery-dataset) which currently only supports tables and views. As soon as external table and materialized view support is added, factory will be enhanced accordingly. + +You can create as many files as you like, the code will loop through it and create the required variables in order to execute everything accordingly. + +## Example + +### Terraform code + +```hcl +module "bq" { + source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric/modules/bigquery-dataset" + + for_each = local.output + project_id = var.project_id + id = each.key + views = try(each.value.views, null) + tables = try(each.value.tables, null) +} +# tftest skip +``` + +### Configuration Structure + +```bash +base_folder +│ +├── tables +│ ├── table_a.yaml +│ ├── table_b.yaml +├── views +│ ├── view_a.yaml +│ ├── view_b.yaml +``` + +## YAML structure and definition formatting + +### Tables + +Table definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following: + +```yaml + +dataset: # required name of the dataset the table is to be placed in +table: # required descriptive name of the table +schema: # required schema in JSON FORMAT Example: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}] +labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"} +use_legacy_sql: boolean # not required, defaults to false +deletion_protection: boolean # not required, defaults to false +``` + +### Views +View definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following: + +```yaml +dataset: # required, name of the dataset the view is to be placed in +view: # required, descriptive name of the view +query: # required, SQL Query for the view in quotes +labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"} +use_legacy_sql: bool # not required, defaults to false +deletion_protection: bool # not required, defaults to false +``` + + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L27) | Project ID | string | ✓ | | +| [tables_dir](variables.tf#L22) | Relative path for the folder storing table data. | string | ✓ | | +| [views_dir](variables.tf#L17) | Relative path for the folder storing view data. | string | ✓ | | + + + + +## TODO + +- [ ] add external table support +- [ ] add materialized view support + diff --git a/blueprints/factories/bigquery-factory/main.tf b/blueprints/factories/bigquery-factory/main.tf new file mode 100644 index 00000000..5995ea19 --- /dev/null +++ b/blueprints/factories/bigquery-factory/main.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + views = { + for f in fileset("${var.views_dir}", "**/*.yaml") : + trimsuffix(f, ".yaml") => yamldecode(file("${var.views_dir}/${f}")) + } + + tables = { + for f in fileset("${var.tables_dir}", "**/*.yaml") : + trimsuffix(f, ".yaml") => yamldecode(file("${var.tables_dir}/${f}")) + } + + output = { + for dataset in distinct([for v in values(merge(local.views, local.tables)) : v.dataset]) : + dataset => { + "views" = { + for k, v in local.views : + v.view => { + friendly_name = v.view + labels = try(v.labels, null) + query = v.query + use_legacy_sql = try(v.use_legacy_sql, false) + deletion_protection = try(v.deletion_protection, false) + } + if v.dataset == dataset + }, + "tables" = { + for k, v in local.tables : + v.table => { + friendly_name = v.table + labels = try(v.labels, null) + options = try(v.options, null) + partitioning = try(v.partitioning, null) + schema = jsonencode(v.schema) + use_legacy_sql = try(v.use_legacy_sql, false) + deletion_protection = try(v.deletion_protection, false) + } + if v.dataset == dataset + } + } + } +} + +module "bq" { + source = "../../../modules/bigquery-dataset" + + for_each = local.output + project_id = var.project_id + id = each.key + views = try(each.value.views, null) + tables = try(each.value.tables, null) +} diff --git a/blueprints/factories/bigquery-factory/variables.tf b/blueprints/factories/bigquery-factory/variables.tf new file mode 100644 index 00000000..e4910d21 --- /dev/null +++ b/blueprints/factories/bigquery-factory/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "views_dir" { + description = "Relative path for the folder storing view data." + type = string +} + +variable "tables_dir" { + description = "Relative path for the folder storing table data." + type = string +} + +variable "project_id" { + description = "Project ID" + type = string + +} diff --git a/tests/blueprints/factories/bigquery_factory/__init__.py b/tests/blueprints/factories/bigquery_factory/__init__.py new file mode 100644 index 00000000..6d6d1266 --- /dev/null +++ b/tests/blueprints/factories/bigquery_factory/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/blueprints/factories/bigquery_factory/fixture/main.tf b/tests/blueprints/factories/bigquery_factory/fixture/main.tf new file mode 100644 index 00000000..75f4fc1c --- /dev/null +++ b/tests/blueprints/factories/bigquery_factory/fixture/main.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "bq" { + source = "../../../../../blueprints/factories/bigquery-factory/" + + project_id = "test-project" + views_dir = "./views" + tables_dir = "./tables" +} diff --git a/tests/blueprints/factories/bigquery_factory/fixture/tables/table_a.yaml b/tests/blueprints/factories/bigquery_factory/fixture/tables/table_a.yaml new file mode 100644 index 00000000..05adbcb0 --- /dev/null +++ b/tests/blueprints/factories/bigquery_factory/fixture/tables/table_a.yaml @@ -0,0 +1,17 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dataset: dataset_a +table: table_a +schema: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}] diff --git a/tests/blueprints/factories/bigquery_factory/fixture/variables.tf b/tests/blueprints/factories/bigquery_factory/fixture/variables.tf new file mode 100644 index 00000000..8269dbbe --- /dev/null +++ b/tests/blueprints/factories/bigquery_factory/fixture/variables.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "views_dir" { + description = "Relative path for the folder storing view data." + type = string + default = "/views" +} + +variable "tables_dir" { + description = "Relative path for the folder storing table data." + type = string + default = "tables" +} + +variable "project_id" { + description = "Project ID" + type = string + default = "test-project" + +} diff --git a/tests/blueprints/factories/bigquery_factory/fixture/views/view_a.yaml b/tests/blueprints/factories/bigquery_factory/fixture/views/view_a.yaml new file mode 100644 index 00000000..23c41b98 --- /dev/null +++ b/tests/blueprints/factories/bigquery_factory/fixture/views/view_a.yaml @@ -0,0 +1,17 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dataset: dataset_b +view: view_a +query: "SELECT CURRENT_DATE() LIMIT 1" diff --git a/tests/blueprints/factories/bigquery_factory/test_plan.py b/tests/blueprints/factories/bigquery_factory/test_plan.py new file mode 100644 index 00000000..74705e42 --- /dev/null +++ b/tests/blueprints/factories/bigquery_factory/test_plan.py @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_resources(e2e_plan_runner): + "Test that plan works and the numbers of resources is as expected." + modules, resources = e2e_plan_runner() + assert len(modules) > 0 + assert len(resources) > 0