Allow creating repositories in Gitlab via Terraform.

This commit is contained in:
Taneli Leppä 2022-06-21 10:45:27 +02:00
parent e0b123125f
commit c3fdc62ff2
17 changed files with 747 additions and 46 deletions

View File

@ -15,6 +15,15 @@
default:
image:
name: registry.gitlab.com/gitlab-org/terraform-images/releases/1.1
before_script:
- |
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
echo "$CICD_MODULES_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
mkdir -p ~/.ssh
ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
cd "$${TF_ROOT}"
cp -R .tf-setup/. .
variables:
FAST_OUTPUTS_BUCKET: ${outputs_bucket}
@ -43,6 +52,7 @@ cache:
# Configure GCP Auth with Access Token
gcp-auth:
stage: gcp-auth
before_script: []
script:
- |
PAYLOAD="$(cat <<EOF
@ -83,6 +93,7 @@ gcp-auth:
# Downloading from bucket into cache
tf-setup:
stage: tf-setup
before_script: []
script:
- |
mkdir -p .tf-setup
@ -104,13 +115,6 @@ tf-init:
stage: tf-init
script:
- |
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null
mkdir -p ~/.ssh
ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
cd "$${TF_ROOT}"
cp -R .tf-setup/. .
gitlab-terraform init
dependencies:
- gcp-auth
@ -120,13 +124,6 @@ tf-validate:
stage: tf-validate
script:
- |
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null
mkdir -p ~/.ssh
ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
cd "$${TF_ROOT}"
cp -R .tf-setup/. .
gitlab-terraform validate
dependencies:
- gcp-auth
@ -136,13 +133,6 @@ tf-plan:
stage: tf-plan
script:
- |
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
echo "$CICD_MODULES_KEY" | tr -d '\r' | ssh-add - > /dev/null
mkdir -p ~/.ssh
ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
ssh-keyscan gitlab.com | sort -u - ~/.ssh/known_hosts -o ~/.ssh/known_hosts
cd "$${TF_ROOT}"
cp -R .tf-setup/. .
gitlab-terraform plan
gitlab-terraform plan-json
dependencies:

View File

@ -358,10 +358,20 @@ federated_identity_providers = {
github-sample = {
attribute_condition = "attribute.repository_owner==\"my-github-org\""
issuer = "github"
custom_settings = null
}
gitlab-sample = {
attribute_condition = "attribute.namespace_path==\"my-gitlab-org\""
issuer = "gitlab"
custom_settings = null
}
gitlab-ce-sample = {
attribute_condition = "attribute.namespace_path==\"my-gitlab-org\""
issuer = "gitlab"
custom_settings = {
issuer_uri = "https://gitlab.fast.example.com"
allowed_audiences = ["https://gitlab.fast.example.com"]
}
}
}
```
@ -382,6 +392,12 @@ cicd_repositories = {
name = "my-gh-org/fast-bootstrap"
type = "github"
}
cicd = {
branch = null
identity_provider = "github-sample"
name = "my-gh-org/fast-cicd"
type = "github"
}
resman = {
branch = "main"
identity_provider = "github-sample"
@ -395,6 +411,8 @@ The `type` attribute can be set to one of the supported repository types: `githu
Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage.
You can use Terraform to automate creation of the repositories using the `00-cicd` stage.
The remaining configuration is manual, as it regards the repositories themselves:
- create a repository for modules
@ -403,7 +421,7 @@ The remaining configuration is manual, as it regards the repositories themselves
- for GitHub
- create a key pair
- create a [deploy key](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) in the modules repository with the public key
- create a `CICD_MODULES_KEY` secret with the private key in each of the repositories that need to access modules
- create a `CICD_MODULES_KEY` secret with the private key in each of the repositories that need to access modules (for Gitlab, please Base64 encode the private key for masking)
- for Gitlab
- TODO
- for Source Repositories
@ -443,31 +461,31 @@ The remaining configuration is manual, as it regards the repositories themselves
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
| [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | <code title="object&#40;&#123;&#10; id &#61; string&#10; organization_id &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | |
| [organization](variables.tf#L152) | Organization details. | <code title="object&#40;&#123;&#10; domain &#61; string&#10; id &#61; number&#10; customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | |
| [prefix](variables.tf#L167) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | |
| [organization](variables.tf#L162) | Organization details. | <code title="object&#40;&#123;&#10; domain &#61; string&#10; id &#61; number&#10; customer_id &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | |
| [prefix](variables.tf#L177) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ | | |
| [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | <code>string</code> | | <code>null</code> | |
| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10; bootstrap &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; resman &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | |
| [custom_role_names](variables.tf#L77) | Names of custom roles defined at the org level. | <code title="object&#40;&#123;&#10; organization_iam_admin &#61; string&#10; service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; organization_iam_admin &#61; &#34;organizationIamAdmin&#34;&#10; service_project_network_admin &#61; &#34;serviceProjectNetworkAdmin&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [federated_identity_providers](variables.tf#L89) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | <code title="map&#40;object&#40;&#123;&#10; attribute_condition &#61; string&#10; issuer &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [groups](variables.tf#L99) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; gcp-billing-admins &#61; &#34;gcp-billing-admins&#34;,&#10; gcp-devops &#61; &#34;gcp-devops&#34;,&#10; gcp-network-admins &#61; &#34;gcp-network-admins&#34;&#10; gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10; gcp-security-admins &#61; &#34;gcp-security-admins&#34;&#10; gcp-support &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [iam](variables.tf#L113) | Organization-level custom IAM settings in role => [principal] format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [iam_additive](variables.tf#L119) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [log_sinks](variables.tf#L127) | Org-level log sinks, in name => {type, filter} format. | <code title="map&#40;object&#40;&#123;&#10; filter &#61; string&#10; type &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code title="&#123;&#10; audit-logs &#61; &#123;&#10; filter &#61; &#34;logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Factivity&#92;&#34; OR logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Fsystem_event&#92;&#34;&#34;&#10; type &#61; &#34;bigquery&#34;&#10; &#125;&#10; vpc-sc &#61; &#123;&#10; filter &#61; &#34;protoPayload.metadata.&#64;type&#61;&#92;&#34;type.googleapis.com&#47;google.cloud.audit.VpcServiceControlAuditMetadata&#92;&#34;&#34;&#10; type &#61; &#34;bigquery&#34;&#10; &#125;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [outputs_location](variables.tf#L161) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> | | <code>null</code> | |
| [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10; bootstrap &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; cicd &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10; resman &#61; object&#40;&#123;&#10; branch &#61; string&#10; identity_provider &#61; string&#10; name &#61; string&#10; type &#61; string&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | |
| [custom_role_names](variables.tf#L83) | Names of custom roles defined at the org level. | <code title="object&#40;&#123;&#10; organization_iam_admin &#61; string&#10; service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; organization_iam_admin &#61; &#34;organizationIamAdmin&#34;&#10; service_project_network_admin &#61; &#34;serviceProjectNetworkAdmin&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [federated_identity_providers](variables.tf#L95) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | <code title="map&#40;object&#40;&#123;&#10; attribute_condition &#61; string&#10; issuer &#61; string&#10; custom_settings &#61; object&#40;&#123;&#10; issuer_uri &#61; string&#10; allowed_audiences &#61; list&#40;string&#41;&#10; &#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [groups](variables.tf#L109) | Group names to grant organization-level permissions. | <code>map&#40;string&#41;</code> | | <code title="&#123;&#10; gcp-billing-admins &#61; &#34;gcp-billing-admins&#34;,&#10; gcp-devops &#61; &#34;gcp-devops&#34;,&#10; gcp-network-admins &#61; &#34;gcp-network-admins&#34;&#10; gcp-organization-admins &#61; &#34;gcp-organization-admins&#34;&#10; gcp-security-admins &#61; &#34;gcp-security-admins&#34;&#10; gcp-support &#61; &#34;gcp-support&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [iam](variables.tf#L123) | Organization-level custom IAM settings in role => [principal] format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [iam_additive](variables.tf#L129) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> | |
| [log_sinks](variables.tf#L137) | Org-level log sinks, in name => {type, filter} format. | <code title="map&#40;object&#40;&#123;&#10; filter &#61; string&#10; type &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code title="&#123;&#10; audit-logs &#61; &#123;&#10; filter &#61; &#34;logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Factivity&#92;&#34; OR logName:&#92;&#34;&#47;logs&#47;cloudaudit.googleapis.com&#37;2Fsystem_event&#92;&#34;&#34;&#10; type &#61; &#34;bigquery&#34;&#10; &#125;&#10; vpc-sc &#61; &#123;&#10; filter &#61; &#34;protoPayload.metadata.&#64;type&#61;&#92;&#34;type.googleapis.com&#47;google.cloud.audit.VpcServiceControlAuditMetadata&#92;&#34;&#34;&#10; type &#61; &#34;bigquery&#34;&#10; &#125;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [outputs_location](variables.tf#L171) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> | | <code>null</code> | |
## Outputs
| name | description | sensitive | consumers |
|---|---|:---:|---|
| [automation](outputs.tf#L82) | Automation resources. | | |
| [billing_dataset](outputs.tf#L87) | BigQuery dataset prepared for billing export. | | |
| [cicd_repositories](outputs.tf#L92) | CI/CD repository configurations. | | |
| [custom_roles](outputs.tf#L104) | Organization-level custom roles. | | |
| [federated_identity](outputs.tf#L109) | Workload Identity Federation pool and providers. | | |
| [outputs_bucket](outputs.tf#L119) | GCS bucket where generated output files are stored. | | |
| [project_ids](outputs.tf#L124) | Projects created by this stage. | | |
| [providers](outputs.tf#L143) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
| [service_accounts](outputs.tf#L133) | Automation service accounts created by this stage. | | |
| [tfvars](outputs.tf#L152) | Terraform variable files for the following stages. | ✓ | |
| [automation](outputs.tf#L87) | Automation resources. | | |
| [billing_dataset](outputs.tf#L92) | BigQuery dataset prepared for billing export. | | |
| [cicd_repositories](outputs.tf#L97) | CI/CD repository configurations. | | |
| [custom_roles](outputs.tf#L109) | Organization-level custom roles. | | |
| [federated_identity](outputs.tf#L114) | Workload Identity Federation pool and providers. | | |
| [outputs_bucket](outputs.tf#L124) | GCS bucket where generated output files are stored. | | |
| [project_ids](outputs.tf#L129) | Projects created by this stage. | | |
| [providers](outputs.tf#L149) | Terraform provider files for this stage and dependent stages. | ✓ | <code>stage-01</code> |
| [service_accounts](outputs.tf#L138) | Automation service accounts created by this stage. | | |
| [tfvars](outputs.tf#L158) | Terraform variable files for the following stages. | ✓ | |
<!-- END TFDOC -->

View File

@ -121,6 +121,37 @@ module "automation-tf-bootstrap-sa" {
# resource hierarchy stage's bucket and service account
module "automation-tf-cicd-gcs" {
source = "../../../modules/gcs"
project_id = module.automation-project.project_id
name = "iac-core-cicd-0"
prefix = local.prefix
versioning = true
iam = {
"roles/storage.objectAdmin" = [module.automation-tf-cicd-provisioning-sa.iam_email]
}
depends_on = [module.organization]
}
module "automation-tf-cicd-provisioning-sa" {
source = "../../../modules/iam-service-account"
project_id = module.automation-project.project_id
name = "cicd-0"
description = "Terraform stage 1 CICD service account."
prefix = local.prefix
# allow SA used by CI/CD workflow to impersonate this SA
iam = {
"roles/iam.serviceAccountTokenCreator" = compact([
try(module.automation-tf-cicd-sa["cicd"].iam_email, null)
])
}
iam_storage_roles = {
(module.automation-tf-output-gcs.name) = ["roles/storage.admin"]
}
}
# resource hierarchy stage's bucket and service account
module "automation-tf-resman-gcs" {
source = "../../../modules/gcs"
project_id = module.automation-project.project_id

View File

@ -23,20 +23,25 @@ locals {
v != null
&&
(
v.type == "sourcerepo"
try(v.type, null) == "sourcerepo"
||
contains(keys(local.identity_providers), coalesce(v.identity_provider, ":"))
contains(keys(local.identity_providers), coalesce(try(v.identity_provider, null), ":"))
)
&&
fileexists("${path.module}/templates/workflow-${v.type}.yaml")
fileexists(format("${path.module}/templates/workflow-%s.yaml", try(v.type, "")))
)
}
cicd_workflow_providers = {
bootstrap = "00-bootstrap-providers.tf"
cicd = "00-cicd-providers.tf"
resman = "01-resman-providers.tf"
}
cicd_workflow_var_files = {
bootstrap = []
cicd = [
"00-bootstrap.auto.tfvars.json",
"globals.auto.tfvars.json"
]
resman = [
"00-bootstrap.auto.tfvars.json",
"globals.auto.tfvars.json"

View File

@ -19,7 +19,8 @@
locals {
identity_providers = {
for k, v in var.federated_identity_providers : k => merge(
v, lookup(local.identity_providers_defs, v.issuer, {})
v, lookup(local.identity_providers_defs, v.issuer, {}),
{ for kk, vv in lookup(v, "custom_settings", {}) : kk => vv if vv != null }
)
}
identity_providers_defs = {

View File

@ -43,6 +43,11 @@ locals {
name = "bootstrap"
sa = module.automation-tf-bootstrap-sa.email
})
"00-cicd" = templatefile(local._tpl_providers, {
bucket = module.automation-tf-cicd-gcs.name
name = "cicd"
sa = module.automation-tf-cicd-provisioning-sa.email
})
"01-resman" = templatefile(local._tpl_providers, {
bucket = module.automation-tf-resman-gcs.name
name = "resman"
@ -134,6 +139,7 @@ output "service_accounts" {
description = "Automation service accounts created by this stage."
value = {
bootstrap = module.automation-tf-bootstrap-sa.email
cicd = module.automation-tf-cicd-provisioning-sa.email
resman = module.automation-tf-resman-sa.email
}
}

View File

@ -37,6 +37,12 @@ variable "cicd_repositories" {
name = string
type = string
})
cicd = object({
branch = string
identity_provider = string
name = string
type = string
})
resman = object({
branch = string
identity_provider = string
@ -91,6 +97,10 @@ variable "federated_identity_providers" {
type = map(object({
attribute_condition = string
issuer = string
custom_settings = object({
issuer_uri = string
allowed_audiences = list(string)
})
}))
default = {}
nullable = false

View File

@ -0,0 +1,208 @@
# CI/CD bootstrap
The primary purpose of this stage is to set up your CI/CD project structure automatically, with most of the
necessary configuration to run the pipelines out of the box.
## How to run this stage
This stage is meant to be executed after the [bootstrap](../00-bootstrap) stage has run, as it leverages the automation service account and bucket created there.
The entire stage is optional, you may also choose to create your repositories manually.
### Providers configuration
The default way of making sure you have the right permissions, is to use the identity of the service account pre-created for this stage during bootstrap, and that you are a member of the group that can impersonate it via provider-level configuration (`gcp-devops` or `organization-admins`).
To simplify setup, the previous stage pre-configures a valid providers file in its output, and optionally writes it to a local file if the `outputs_location` variable is set to a valid path.
If you have set a valid value for `outputs_location` in the bootstrap stage (see the [bootstrap stage README](../00-bootstrap/#output-files-and-cross-stage-variables) for more details), simply link the relevant `providers.tf` file from this stage's folder in the path you specified:
```bash
# `outputs_location` is set to `~/fast-config`
ln -s ~/fast-config/providers/00-cicd-providers.tf .
```
If you have not configured `outputs_location` in bootstrap, you can derive the providers file from that stage's outputs:
```bash
cd ../00-bootstrap
terraform output -json providers | jq -r '.["00-cicd"]' \
> ../00-cicd/providers.tf
```
If you want to continue to rely on `outputs_location` logic, create a `terraform.tfvars` file and configure it as described [here](../00-bootstrap/#output-files-and-cross-stage-variables).
### Variable configuration
There are two broad sets of variables you will need to fill in:
- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.)
- variables specific to resources managed by this stage
To avoid the tedious job of filling in the first group of variable with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files.
If you configured a valid path for `outputs_location` in the bootstrap stage, simply link the relevant `terraform-*.auto.tfvars.json` files from the outputs folder. For this stage, you need the `.tfvars` file compiled manually for the bootstrap stage, and the one generated by it:
```bash
# `outputs_location` is set to `~/fast-config`
ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json .
# also copy the tfvars file used for the bootstrap stage
cp ../00-bootstrap/terraform.tfvars .
```
A second set of variables is specific to this stage, they are all optional so if you need to customize them, create an extra `terraform.tfvars` file or add them to the file copied from bootstrap.
Refer to the [Variables](#variables) table at the bottom of this document, for a full list of variables, their origin (e.g. a stage or specific to this one), and descriptions explaining their meaning. The sections below also describe some of the possible customizations.
### CI/CD systems
#### Gitlab
To configure Gitlab, add the following variable:
```hcl
gitlab = {
url = "https://gitlab.com" # Or self-hosted URL
project_visibility = "private"
shared_runners_enabled = true
}
```
Also set `GITLAB_TOKEN` to a token that has appropriate permissions.
#### GitHub
To configure GitHub, add the following variable:
```hcl
github = {
url = null # Or GitHub Enterprise base URL
visibility = "private"
}
```
Also set `GITHUB_TOKEN` to a token that has appropriate permissions.
### CI/CD repositories
While the other stages create the necessary supporting structure for their CI/CD pipelines, like service accounts
and such, the `00-cicd` stage creates all the repositories in your CI/CD system through automation. Its
configuration is essentially a combination of all the `cicd_repositories` variables of the other stages
plus additional CI/CD system specific configuration information.
This is an example of configuring the repositories in this stage.
```hcl
cicd_repositories = {
bootstrap = {
branch = null
identity_provider = "github-sample"
name = "my-gh-org/fast-bootstrap"
description = "Google Cloud bootstrapping"
type = "github"
create = true
create_group = true
}
cicd = {
branch = null
identity_provider = "github-sample"
name = "my-gh-org/fast-cicd"
description = "Fabric FAST CI/CD setup"
type = "github"
create = true
create_group = true
}
resman = {
branch = "main"
identity_provider = "github-sample"
name = "my-gh-org/fast-resman"
description = "Google Cloud resource management"
type = "github"
create = true
create_group = true
}
networking = {
branch = "main"
identity_provider = "github-sample"
name = "my-gh-org/fast-networking"
description = "Google Cloud networking setup"
type = "github"
create = true
create_group = true
}
security = {
branch = "main"
identity_provider = "github-sample"
description = "Google Cloud security settings"
name = "my-gh-org/fast-security"
type = "github"
create = true
create_group = true
}
data-platform = {
branch = "main"
identity_provider = "github-sample"
name = "my-gh-org/fast-data-platform"
description = "Google Cloud data platform"
type = "github"
create = true
create_group = true
}
project-factory = {
branch = "main"
identity_provider = "github-sample"
name = "my-gh-org/fast-project-factory"
description = "Google Cloud project factory"
type = "github"
create = true
create_group = true
}
}
```
The `type` attribute can be set to one of the supported repository types: `github` or `gitlab`.
Once the stage is applied the generated output files will contain pre-configured workflow files for each repository, that will use Workload Identity Federation via a dedicated service account for each repository to impersonate the automation service account for the stage.
Once done, you can run this stage:
```bash
terraform init
terraform apply
```
<!-- TFDOC OPTS files:1 show_extra:1 -->
<!-- BEGIN TFDOC -->
## Files
| name | description | resources |
|---|---|---|
| [cicd.tf](./cicd.tf) | None | <code>tls_private_key</code> |
| [github.tf](./github.tf) | None | <code>github_actions_secret</code> · <code>github_repository</code> |
| [gitlab.tf](./gitlab.tf) | None | <code>gitlab_group</code> · <code>gitlab_project</code> · <code>gitlab_project_variable</code> |
| [main.tf](./main.tf) | Module-level locals and resources. | |
| [outputs-files.tf](./outputs-files.tf) | Output files persistence to local filesystem. | <code>local_file</code> |
| [outputs-gcs.tf](./outputs-gcs.tf) | Output files persistence to automation GCS bucket. | <code>google_storage_bucket_object</code> |
| [outputs.tf](./outputs.tf) | Module outputs. | |
| [variables.tf](./variables.tf) | Module variables. | |
| [versions.tf](./versions.tf) | Version pins. | |
## Variables
| name | description | type | required | default | producer |
|---|---|:---:|:---:|:---:|:---:|
| [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | <code title="object&#40;&#123;&#10; outputs_bucket &#61; string&#10; project_id &#61; string&#10; project_number &#61; string&#10; federated_identity_pool &#61; string&#10; federated_identity_providers &#61; map&#40;object&#40;&#123;&#10; issuer &#61; string&#10; issuer_uri &#61; string&#10; name &#61; string&#10; principal_tpl &#61; string&#10; principalset_tpl &#61; string&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | | <code>00-bootstrap</code> |
| [cicd_repositories](variables.tf#L35) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | <code title="object&#40;&#123;&#10; bootstrap &#61; object&#40;&#123;&#10; branch &#61; string&#10; name &#61; string&#10; description &#61; string&#10; type &#61; string&#10; create &#61; bool&#10; create_group &#61; bool&#10; &#125;&#41;&#10; resman &#61; object&#40;&#123;&#10; branch &#61; string&#10; name &#61; string&#10; description &#61; string&#10; type &#61; string&#10; create &#61; bool&#10; create_group &#61; bool&#10; &#125;&#41;&#10; networking &#61; object&#40;&#123;&#10; branch &#61; string&#10; name &#61; string&#10; description &#61; string&#10; type &#61; string&#10; create &#61; bool&#10; create_group &#61; bool&#10; &#125;&#41;&#10; security &#61; object&#40;&#123;&#10; branch &#61; string&#10; name &#61; string&#10; description &#61; string&#10; type &#61; string&#10; create &#61; bool&#10; create_group &#61; bool&#10; &#125;&#41;&#10; data-platform &#61; object&#40;&#123;&#10; branch &#61; string&#10; name &#61; string&#10; description &#61; string&#10; type &#61; string&#10; create &#61; bool&#10; create_group &#61; bool&#10; &#125;&#41;&#10; project-factory &#61; object&#40;&#123;&#10; branch &#61; string&#10; name &#61; string&#10; description &#61; string&#10; type &#61; string&#10; create &#61; bool&#10; create_group &#61; bool&#10; &#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | |
| [custom_roles](variables.tf#L132) | Custom roles defined at the org level, in key => id format. | <code title="object&#40;&#123;&#10; service_project_network_admin &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> | <code>00-bootstrap</code> |
| [github](variables.tf#L120) | GitHub settings | <code title="object&#40;&#123;&#10; url &#61; string&#10; visibility &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; url &#61; null&#10; visibility &#61; &#34;private&#34;&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [gitlab](variables.tf#L106) | Gitlab settings | <code title="object&#40;&#123;&#10; url &#61; string&#10; project_visibility &#61; string&#10; shared_runners_enabled &#61; bool&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; url &#61; &#34;https:&#47;&#47;gitlab.com&#34;&#10; project_visibility &#61; &#34;private&#34;&#10; shared_runners_enabled &#61; true&#10;&#125;">&#123;&#8230;&#125;</code> | |
| [outputs_location](variables.tf#L141) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | <code>string</code> | | <code>null</code> | |
## Outputs
| name | description | sensitive | consumers |
|---|---|:---:|---|
| [tfvars](outputs.tf#L30) | Terraform variable files for the following stages. | ✓ | |
<!-- END TFDOC -->

View File

@ -0,0 +1,38 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
supported_cicd_systems = ["gitlab", "github", "sourcerepo"]
cicd_repositories = {
for k, v in coalesce(var.cicd_repositories, {}) : k => merge(v,
{ group = join("/", slice(split("/", v.name), 0, length(split("/", v.name)) - 1)) },
{ name = element(split("/", v.name), length(split("/", v.name)) - 1) },
{ create_group = try(v.create_group, true) })
if(
v != null
&&
contains(local.supported_cicd_systems, try(v.type, ""))
)
}
cicd_repositories_by_system = { for system in local.supported_cicd_systems : system => {
for k, v in local.cicd_repositories : k => v if v.type == system
}
}
}
resource "tls_private_key" "cicd-modules-key" {
algorithm = "ED25519"
}

View File

@ -0,0 +1,51 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
github_groups = distinct([for k, v in local.cicd_repositories_by_system["github"] : v.group])
}
provider "github" {
base_url = var.github.url
owner = local.github_groups[0]
}
data "github_organization" "organization" {
for_each = toset(local.github_groups)
name = each.value
}
data "github_repository" "repositories" {
for_each = { for name, repo in local.cicd_repositories_by_system["github"] : name => repo if !try(repo.create, true) }
full_name = format("%s/%s", each.value.group, each.value.name)
}
resource "github_repository" "repositories" {
for_each = { for name, repo in local.cicd_repositories_by_system["github"] : name => repo if try(repo.create, true) }
name = each.value.name
description = each.value.description
visibility = var.github.visibility
}
resource "github_actions_secret" "actions-modules-key" {
for_each = { for name, repo in local.cicd_repositories_by_system["github"] : name => repo }
repository = try(each.value.create, true) ? github_repository.repositories[each.key].name : data.github_repository.repositories[each.key].name
secret_name = "CICD_MODULES_KEY"
plaintext_value = tls_private_key.cicd-modules-key.private_key_openssh
}

View File

@ -0,0 +1,70 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
gitlab_create_groups = distinct([for k, v in local.cicd_repositories_by_system["gitlab"] : v.group if try(v.create_group, false)])
gitlab_existing_groups = distinct([for k, v in local.cicd_repositories_by_system["gitlab"] : v.group if !try(v.create_group, false)])
}
provider "gitlab" {
base_url = var.gitlab.url
}
data "gitlab_group" "group" {
for_each = toset(local.gitlab_existing_groups)
full_path = each.value
}
data "gitlab_project" "projects" {
for_each = { for name, repo in local.cicd_repositories_by_system["gitlab"] : name => repo if !try(repo.create, true) }
id = format("%s/%s", each.value.group, each.value.name)
}
resource "gitlab_group" "group" {
for_each = toset(local.gitlab_create_groups)
name = each.value
path = each.value
description = "Cloud Foundation Fabric FAST: github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/fast/"
}
resource "gitlab_project" "projects" {
for_each = { for name, repo in local.cicd_repositories_by_system["gitlab"] : name => repo if try(repo.create, true) }
name = each.value.name
namespace_id = each.value.create_group ? gitlab_group.group[each.value.group].id : data.gitlab_group.group[each.value.group].id
description = each.value.description
visibility_level = var.gitlab.project_visibility
shared_runners_enabled = var.gitlab.shared_runners_enabled
auto_devops_enabled = false
}
resource "gitlab_project_variable" "project-modules-key" {
for_each = { for name, repo in local.cicd_repositories_by_system["gitlab"] : name => repo }
project = try(each.value.create, true) ? gitlab_project.projects[each.key].id : data.gitlab_project.projects[each.key].id
key = "CICD_MODULES_KEY"
value = base64encode(tls_private_key.cicd-modules-key.private_key_openssh)
protected = false
masked = true
variable_type = "env_var"
}

View File

@ -0,0 +1,16 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

View File

@ -0,0 +1,24 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# tfdoc:file:description Output files persistence to local filesystem.
resource "local_file" "tfvars" {
for_each = var.outputs_location == null ? {} : { 1 = 1 }
file_permission = "0644"
filename = "${pathexpand(var.outputs_location)}/tfvars/00-cicd.auto.tfvars.json"
content = jsonencode(local.tfvars)
}

View File

@ -0,0 +1,23 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# tfdoc:file:description Output files persistence to automation GCS bucket.
resource "google_storage_bucket_object" "tfvars" {
bucket = var.automation.outputs_bucket
name = "tfvars/00-bootstrap.auto.tfvars.json"
content = jsonencode(local.tfvars)
}

View File

@ -0,0 +1,34 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
gitlab_cicd_https = { for k, v in local.cicd_repositories_by_system["gitlab"] : k => v.create ? gitlab_project.projects[k].http_url_to_repo : data.gitlab_project.projects[k].http_url_to_repo }
gitlab_cicd_ssh = { for k, v in local.cicd_repositories_by_system["gitlab"] : k => v.create ? gitlab_project.projects[k].ssh_url_to_repo : data.gitlab_project.projects[k].ssh_url_to_repo }
github_cicd_https = { for k, v in local.cicd_repositories_by_system["github"] : k => v.create ? github_repository.repositories[k].http_clone_url : data.github_repository.repositories[k].http_clone_url }
github_cicd_ssh = { for k, v in local.cicd_repositories_by_system["github"] : k => v.create ? github_repository.repositories[k].git_clone_url : data.github_repository.repositories[k].git_clone_url }
tfvars = {
cicd_repositories = merge(local.cicd_repositories_by_system["gitlab"], local.cicd_repositories_by_system["github"])
cicd_ssh_urls = merge(local.gitlab_cicd_ssh, local.github_cicd_ssh)
cicd_https_urls = merge(local.gitlab_cicd_https, local.gitlab_cicd_https)
}
}
output "tfvars" {
description = "Terraform variable files for the following stages."
sensitive = true
value = local.tfvars
}

View File

@ -0,0 +1,145 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "automation" {
# tfdoc:variable:source 00-bootstrap
description = "Automation resources created by the bootstrap stage."
type = object({
outputs_bucket = string
project_id = string
project_number = string
federated_identity_pool = string
federated_identity_providers = map(object({
issuer = string
issuer_uri = string
name = string
principal_tpl = string
principalset_tpl = string
}))
})
}
variable "cicd_repositories" {
description = "CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed."
type = object({
bootstrap = object({
branch = string
name = string
description = string
type = string
create = bool
create_group = bool
})
resman = object({
branch = string
name = string
description = string
type = string
create = bool
create_group = bool
})
networking = object({
branch = string
name = string
description = string
type = string
create = bool
create_group = bool
})
security = object({
branch = string
name = string
description = string
type = string
create = bool
create_group = bool
})
data-platform = object({
branch = string
name = string
description = string
type = string
create = bool
create_group = bool
})
project-factory = object({
branch = string
name = string
description = string
type = string
create = bool
create_group = bool
})
})
default = null
validation {
condition = alltrue([
for k, v in coalesce(var.cicd_repositories, {}) :
v == null || try(v.name, null) != null
])
error_message = "Non-null repositories need a non-null name."
}
validation {
condition = alltrue([
for k, v in coalesce(var.cicd_repositories, {}) :
v == null || (
contains(["github", "gitlab", "sourcerepo"], coalesce(try(v.type, null), "null"))
)
])
error_message = "Invalid repository type, supported types: 'github' 'gitlab' or 'sourcerepo'."
}
}
variable "gitlab" {
description = "Gitlab settings"
type = object({
url = string
project_visibility = string
shared_runners_enabled = bool
})
default = {
url = "https://gitlab.com"
project_visibility = "private"
shared_runners_enabled = true
}
}
variable "github" {
description = "GitHub settings"
type = object({
url = string
visibility = string
})
default = {
url = null
visibility = "private"
}
}
variable "custom_roles" {
# tfdoc:variable:source 00-bootstrap
description = "Custom roles defined at the org level, in key => id format."
type = object({
service_project_network_admin = string
})
default = null
}
variable "outputs_location" {
description = "Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable"
type = string
default = null
}

View File

@ -0,0 +1,31 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
terraform {
required_version = ">= 1.1.0"
required_providers {
gitlab = {
source = "gitlabhq/gitlab"
version = ">= 3.15.0"
}
github = {
source = "integrations/github"
version = ">= 4.26.0"
}
tls = {
source = "hashicorp/tls"
version = "3.4.0"
}
}
}