Marcwo/bqfactory (#884)
* adding the bq factory Co-authored-by: Ludovico Magnocavallo <ludomagno@google.com>
This commit is contained in:
parent
d0ccd9ac31
commit
5ef93e2829
|
@ -0,0 +1,88 @@
|
|||
# Google Cloud BQ Factory
|
||||
|
||||
This module allows creation and management of BigQuery datasets and views as well as tables by defining them in well formatted `yaml` files.
|
||||
|
||||
Yaml abstraction for BQ can simplify users onboarding and also makes creation of tables easier compared to HCL.
|
||||
|
||||
Subfolders distinguish between views and tables and ensures easier navigation for users.
|
||||
|
||||
This factory is based on the [BQ dataset module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/bigquery-dataset) which currently only supports tables and views. As soon as external table and materialized view support is added, factory will be enhanced accordingly.
|
||||
|
||||
You can create as many files as you like, the code will loop through it and create the required variables in order to execute everything accordingly.
|
||||
|
||||
## Example
|
||||
|
||||
### Terraform code
|
||||
|
||||
```hcl
|
||||
module "bq" {
|
||||
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric/modules/bigquery-dataset"
|
||||
|
||||
for_each = local.output
|
||||
project_id = var.project_id
|
||||
id = each.key
|
||||
views = try(each.value.views, null)
|
||||
tables = try(each.value.tables, null)
|
||||
}
|
||||
# tftest skip
|
||||
```
|
||||
|
||||
### Configuration Structure
|
||||
|
||||
```bash
|
||||
base_folder
|
||||
│
|
||||
├── tables
|
||||
│ ├── table_a.yaml
|
||||
│ ├── table_b.yaml
|
||||
├── views
|
||||
│ ├── view_a.yaml
|
||||
│ ├── view_b.yaml
|
||||
```
|
||||
|
||||
## YAML structure and definition formatting
|
||||
|
||||
### Tables
|
||||
|
||||
Table definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following:
|
||||
|
||||
```yaml
|
||||
|
||||
dataset: # required name of the dataset the table is to be placed in
|
||||
table: # required descriptive name of the table
|
||||
schema: # required schema in JSON FORMAT Example: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}]
|
||||
labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"}
|
||||
use_legacy_sql: boolean # not required, defaults to false
|
||||
deletion_protection: boolean # not required, defaults to false
|
||||
```
|
||||
|
||||
### Views
|
||||
View definition to be placed in a set of yaml files in the corresponding subfolder. Structure should look as following:
|
||||
|
||||
```yaml
|
||||
dataset: # required, name of the dataset the view is to be placed in
|
||||
view: # required, descriptive name of the view
|
||||
query: # required, SQL Query for the view in quotes
|
||||
labels: # not required, defaults to {}, Example: {"a":"thisislabela","b":"thisislabelb"}
|
||||
use_legacy_sql: bool # not required, defaults to false
|
||||
deletion_protection: bool # not required, defaults to false
|
||||
```
|
||||
|
||||
<!-- BEGIN TFDOC -->
|
||||
|
||||
## Variables
|
||||
|
||||
| name | description | type | required | default |
|
||||
|---|---|:---:|:---:|:---:|
|
||||
| [project_id](variables.tf#L27) | Project ID | <code>string</code> | ✓ | |
|
||||
| [tables_dir](variables.tf#L22) | Relative path for the folder storing table data. | <code>string</code> | ✓ | |
|
||||
| [views_dir](variables.tf#L17) | Relative path for the folder storing view data. | <code>string</code> | ✓ | |
|
||||
|
||||
<!-- END TFDOC -->
|
||||
|
||||
|
||||
## TODO
|
||||
|
||||
- [ ] add external table support
|
||||
- [ ] add materialized view support
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
locals {
|
||||
views = {
|
||||
for f in fileset("${var.views_dir}", "**/*.yaml") :
|
||||
trimsuffix(f, ".yaml") => yamldecode(file("${var.views_dir}/${f}"))
|
||||
}
|
||||
|
||||
tables = {
|
||||
for f in fileset("${var.tables_dir}", "**/*.yaml") :
|
||||
trimsuffix(f, ".yaml") => yamldecode(file("${var.tables_dir}/${f}"))
|
||||
}
|
||||
|
||||
output = {
|
||||
for dataset in distinct([for v in values(merge(local.views, local.tables)) : v.dataset]) :
|
||||
dataset => {
|
||||
"views" = {
|
||||
for k, v in local.views :
|
||||
v.view => {
|
||||
friendly_name = v.view
|
||||
labels = try(v.labels, null)
|
||||
query = v.query
|
||||
use_legacy_sql = try(v.use_legacy_sql, false)
|
||||
deletion_protection = try(v.deletion_protection, false)
|
||||
}
|
||||
if v.dataset == dataset
|
||||
},
|
||||
"tables" = {
|
||||
for k, v in local.tables :
|
||||
v.table => {
|
||||
friendly_name = v.table
|
||||
labels = try(v.labels, null)
|
||||
options = try(v.options, null)
|
||||
partitioning = try(v.partitioning, null)
|
||||
schema = jsonencode(v.schema)
|
||||
use_legacy_sql = try(v.use_legacy_sql, false)
|
||||
deletion_protection = try(v.deletion_protection, false)
|
||||
}
|
||||
if v.dataset == dataset
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module "bq" {
|
||||
source = "../../../modules/bigquery-dataset"
|
||||
|
||||
for_each = local.output
|
||||
project_id = var.project_id
|
||||
id = each.key
|
||||
views = try(each.value.views, null)
|
||||
tables = try(each.value.tables, null)
|
||||
}
|
|
@ -0,0 +1,31 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "views_dir" {
|
||||
description = "Relative path for the folder storing view data."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "tables_dir" {
|
||||
description = "Relative path for the folder storing table data."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Project ID"
|
||||
type = string
|
||||
|
||||
}
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
|
@ -0,0 +1,23 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
module "bq" {
|
||||
source = "../../../../../blueprints/factories/bigquery-factory/"
|
||||
|
||||
project_id = "test-project"
|
||||
views_dir = "./views"
|
||||
tables_dir = "./tables"
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
dataset: dataset_a
|
||||
table: table_a
|
||||
schema: [{name: "test", type: "STRING"},{name: "test2", type: "INT64"}]
|
|
@ -0,0 +1,34 @@
|
|||
/**
|
||||
* Copyright 2022 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
variable "views_dir" {
|
||||
description = "Relative path for the folder storing view data."
|
||||
type = string
|
||||
default = "/views"
|
||||
}
|
||||
|
||||
variable "tables_dir" {
|
||||
description = "Relative path for the folder storing table data."
|
||||
type = string
|
||||
default = "tables"
|
||||
}
|
||||
|
||||
variable "project_id" {
|
||||
description = "Project ID"
|
||||
type = string
|
||||
default = "test-project"
|
||||
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
dataset: dataset_b
|
||||
view: view_a
|
||||
query: "SELECT CURRENT_DATE() LIMIT 1"
|
|
@ -0,0 +1,19 @@
|
|||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
def test_resources(e2e_plan_runner):
|
||||
"Test that plan works and the numbers of resources is as expected."
|
||||
modules, resources = e2e_plan_runner()
|
||||
assert len(modules) > 0
|
||||
assert len(resources) > 0
|
Loading…
Reference in New Issue