From 7f7786ae9cf859e94c287936f8c2ea328dc41003 Mon Sep 17 00:00:00 2001 From: "a@a.ru" Date: Wed, 5 Jun 2019 16:04:33 +0300 Subject: [PATCH] Fix a number of variables --- .gitignore | 2 + attach_existing_rds.yml | 2 +- clean.yml | 18 ++- deploy_infra.yml | 5 +- deploy_software.yml | 6 +- destroy.yml | 3 +- host_vars/all.yml.example | 4 +- roles/attach_existing_rds/tasks/main.yml | 2 - roles/check/tasks/main.yml | 24 ++- roles/destroy/tasks/main.yml | 24 +-- roles/destroy/tasks/parameter_store.yml | 6 +- roles/dynamodb/tasks/main.yml | 4 +- roles/main_infra/defaults/main.yml | 1 + roles/main_infra/tasks/main.yml | 85 ++++++---- roles/main_infra/tasks/parameter_store.yml | 2 +- roles/main_infra/templates/hosts.tf.j2 | 32 ++-- roles/main_infra/templates/routing.tf.j2 | 4 +- .../main_infra/templates/terraform.tfvars.j2 | 20 +-- roles/main_software/tasks/main.yml | 149 ++++++++++-------- roles/s3_config/tasks/main.yml | 2 +- 20 files changed, 221 insertions(+), 174 deletions(-) diff --git a/.gitignore b/.gitignore index 29fb423..4cf995b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ log.txt # Terraform State *.terraform* +*.tfstate *terraform.tfstate.d* *tfplan* roles/main_infra/files/backend.tfvars @@ -33,5 +34,6 @@ host_vars/* .*.swp blockscout-*/ +roles/main_infra/files-* hosts diff --git a/attach_existing_rds.yml b/attach_existing_rds.yml index 2675396..050d48e 100644 --- a/attach_existing_rds.yml +++ b/attach_existing_rds.yml @@ -9,7 +9,7 @@ with_items: - s3 - dynamodb - when: backend|bool == true + when: backend | bool - include_role: name: attach_existing_rds always: diff --git a/clean.yml b/clean.yml index c1e664a..e1e19d7 100644 --- a/clean.yml +++ b/clean.yml @@ -1,14 +1,16 @@ - name: Clean TF cache - hosts: localhost + hosts: localhost,all tasks: - name: Clean TF cache file: state: absent path: "{{ item }}" - with_items: - - roles/main_infra/files/.terraform - - roles/main_infra/files/terraform.tfstate.d - - roles/main_infra/files/main.tfvars - - roles/main_infra/files/backend.tfvars - - roles/main_infra/files/terraform.tfplan - - log.txt + with_fileglob: + - "roles/main_infra/files/.terraform" + - "roles/main_infra/files/terraform.tfstate.d" + - "roles/main_infra/files/main.tfvars" + - "roles/main_infra/files/backend.tfvars" + - "roles/main_infra/files/terraform.tfplan" + - "log.txt" + - "blockscout-*" + - "/tmp/files-*" diff --git a/deploy_infra.yml b/deploy_infra.yml index 5d64e04..0d2a644 100644 --- a/deploy_infra.yml +++ b/deploy_infra.yml @@ -1,5 +1,5 @@ - name: Prepare infrastructure - hosts: localhost + hosts: all tasks: - block: - include_role: @@ -9,9 +9,10 @@ with_items: - s3 - dynamodb - when: backend|bool == true + when: backend | bool - include_role: name: main_infra + when: inventory_hostname == groups[group_names[0]][0] always: - include_role: name: s3_config diff --git a/deploy_software.yml b/deploy_software.yml index d3f5213..965041a 100644 --- a/deploy_software.yml +++ b/deploy_software.yml @@ -8,10 +8,10 @@ always: - include_role: name: s3 - when: backend|bool == true and (upload_debug_info_to_s3|bool == true or upload_config_to_s3|bool ==true) + when: backend|bool and (upload_debug_info_to_s3|bool or upload_config_to_s3|bool) - include_role: name: s3_config - when: backend|bool == true and upload_config_to_s3|bool == true + when: backend|bool and upload_config_to_s3|bool - include_role: name: s3_debug - when: backend|bool == true and upload_debug_info_to_s3|bool == true + when: backend|bool and upload_debug_info_to_s3|bool diff --git a/destroy.yml b/destroy.yml index e39b036..e4eea51 100644 --- a/destroy.yml +++ b/destroy.yml @@ -1,5 +1,6 @@ - name: Destroy infrastructure - hosts: localhost + hosts: all + serial: 1 roles: - { role: destroy, when: "confirmation|bool == True" } vars_prompt: diff --git a/host_vars/all.yml.example b/host_vars/all.yml.example index e83cb6c..3177452 100644 --- a/host_vars/all.yml.example +++ b/host_vars/all.yml.example @@ -22,8 +22,8 @@ env_vars: #ALB_CERTIFICATE_ARN: "arn:aws:acm:us-east-1:290379793816:certificate/6d1bab74-fb46-4244-aab2-832bf519ab24" #ARN of the certificate to attach to the LB. Required if ECTO_USE_SSL is set to true #HEART_BEAT_TIMEOUT: 30 # Heartbeat is an Erlang monitoring service that will restart BlockScout if it becomes unresponsive. This variables configures the timeout before Blockscout will be restarted. #HEART_COMMAND: "sudo systemctl restart explorer.service" # This variable represents a command that is used to restart the service - BLOCKSCOUT_VERSION: "v1.3.13-beta" # Added to the footer to signify the current BlockScout version - RELEASE_LINK: "https://github.com/poanetwork/blockscout/releases/tag/v1.3.13-beta" # The link to Blockscout release notes in the footer. + #BLOCKSCOUT_VERSION: "v1.3.13-beta" # Added to the footer to signify the current BlockScout version + #RELEASE_LINK: "https://github.com/poanetwork/blockscout/releases/tag/v1.3.13-beta" # The link to Blockscout release notes in the footer. #ELIXIR_VERSION: "v1.8.1" # Elixir version to install on the node before Blockscout deploy #BLOCK_TRANSFORMER: "base" # Transformer for blocks: base or clique. #GRAPHIQL_TRANSACTION: "0xbc426b4792c48d8ca31ec9786e403866e14e7f3e4d39c7f2852e518fae529ab4" # Random tx hash on the network, used as default for graphiql tx. diff --git a/roles/attach_existing_rds/tasks/main.yml b/roles/attach_existing_rds/tasks/main.yml index 3b7a9df..9a080d2 100644 --- a/roles/attach_existing_rds/tasks/main.yml +++ b/roles/attach_existing_rds/tasks/main.yml @@ -16,8 +16,6 @@ template: src: roles/main_infra/templates/terraform.tfvars.j2 dest: roles/main_infra/files/terraform.tfvars - vars: - db_iops: "{{ chain_db_iops | default({}) }}" - name: Generating backend file template: diff --git a/roles/check/tasks/main.yml b/roles/check/tasks/main.yml index 2792753..cee6df3 100644 --- a/roles/check/tasks/main.yml +++ b/roles/check/tasks/main.yml @@ -1,33 +1,27 @@ - name: Check prefix fail: - msg: "The prefix '{{ prefix }}' is invalid. It must consist only of the lowercase characters a-z and digits 0-9, and must be between 3 and 5 characters long." - when: prefix|length < 3 or prefix|length > 5 or prefix is not match("^[a-z0-9]+$") + msg: "The prefix '{{ group_names[0] }}' is invalid. It must consist only of the lowercase characters a-z and digits 0-9, and must be between 3 and 5 characters long." + when: group_names[0] | length < 3 or group_names[0] | length > 5 or group_names[0] is not match("^[a-z0-9]+$") - name: Check chain names fail: - msg: "The prefix '{{ item }}' is invalid. It must consist only of the lowercase characters a-z and digits 0-9, and must not more than 5 characters long." - when: item.key|length > 5 or item.key is not match("^[a-z0-9]+$") - with_dict: "{{ chain_custom_environment }}" + msg: "The chain '{{ item }}' is invalid. It must consist only of the lowercase characters a-z and digits 0-9, and must not more than 5 characters long." + when: (item.key | length > 5 or item.key is not match("^[a-z0-9]+$")) and item.key != "all" and item.key != "ungrouped" + with_dict: "{{ groups }}" - name: Check if terraform is installed - command: which terraform + command: "{{ terraform_location }} --version" register: terraform_status changed_when: false -- name: Terraform check result - fail: - msg: "Terraform is not installed" - when: terraform_status.stdout == "" - - name: Check if python is installed - command: which python - register: python_status + command: "{{ ansible_python_interpreter }} --version" changed_when: false - name: Python check result fail: - msg: "Python either is not installed or is too old. Please install python version 2.6 or higher" - when: python_status.stdout == "" or python_int_version|int < 260 + msg: "Python is too old. Please install python version 2.6 or higher" + when: python_int_version | int < 260 vars: python_int_version: "{{ ansible_python_version.split('.')[0]|int * 100 + ansible_python_version.split('.')[1]|int * 10 + ansible_python_version.split('.')[2]|int }}" diff --git a/roles/destroy/tasks/main.yml b/roles/destroy/tasks/main.yml index 489b1a7..dd741b0 100644 --- a/roles/destroy/tasks/main.yml +++ b/roles/destroy/tasks/main.yml @@ -10,26 +10,24 @@ state: absent dest: roles/main_infra/files/remote-backend-selector.tf when: - - backend | default ('false') | bool != true + - not backend | default ('false') | bool - name: Generating variables file template: src: roles/main_infra/templates/terraform.tfvars.j2 dest: roles/main_infra/files/terraform.tfvars - vars: - db_iops: "{{ chain_db_iops | default({}) }}" - name: Generating backend file template: src: roles/main_infra/templates/backend.tfvars.j2 dest: roles/main_infra/files/backend.tfvars - when: backend|bool == true + when: backend | bool - name: Generate Terraform files template: src: "{{ item.key }}" dest: "{{ item.value }}" - with_dict: {roles/main_infra/templates/hosts.tf.j2: roles/main_infra/files/hosts.tf,roles/main_infra/templates/routing.tf.j2: roles/main_infra/files/routing.tf,roles/main_infra/templates/provider.tf.j2: roles/main_infra/files/provider.tf} + with_dict: { roles/main_infra/templates/hosts.tf.j2: roles/main_infra/files/hosts.tf, roles/main_infra/templates/routing.tf.j2: roles/main_infra/files/routing.tf, roles/main_infra/templates/provider.tf.j2: roles/main_infra/files/provider.tf } # This is due to the TF0.11 bug which do not allow to completely destroy resources if interpolation syntax is used in outputs.tf at edge cases - name: Check if outputs.tf exists @@ -49,7 +47,7 @@ file: path: roles/main_infra/files/.terraform/ state: absent - when: stat_result.stat.exists == True + when: stat_result.stat.exists - name: Terraform destroy main infra shell: "echo yes | {{ terraform_location }} {{ item }}" @@ -61,10 +59,6 @@ - name: Delete vars from parameter store include: parameter_store.yml - loop: "{{ chain_custom_environment.keys() }}" - loop_control: - loop_var: chain - index_var: index - name: Check if outputs.tf.backup exists stat: path=roles/main_infra/files/outputs.tf.backup @@ -78,6 +72,12 @@ pause: prompt: "Do you want to delete S3 bucket with state file and DynamoDB attached to it also? [Yes/No] Default: No" register: user_answer + until: user_answer.user_input | lower in conditional + retries: 10000 + delay: 1 + vars: + conditional: ['yes','no','true','false'] + when: inventory_hostname == groups['all'][0] - name: Destroy S3 bucket s3_bucket: @@ -93,7 +93,7 @@ secret_key: "{{ aws_secret_key|default(omit) }}" profile: "{{ aws_profile|default(omit) }}" region: "{{ aws_region|default(omit) }}" - when: user_answer.user_input|bool == True + when: hostvars[groups['all'][0]].user_answer.user_input | bool - dynamodb_table: name: "{{ prefix }}-{{ dynamodb_table }}" @@ -107,4 +107,4 @@ secret_key: "{{ aws_secret_key|default(omit) }}" profile: "{{ aws_profile|default(omit) }}" region: "{{ aws_region|default(omit) }}" - when: user_answer.user_input|bool == True + when: hostvars[groups['all'][0]].user_answer.user_input | bool diff --git a/roles/destroy/tasks/parameter_store.yml b/roles/destroy/tasks/parameter_store.yml index 5c7ec27..d1c56de 100644 --- a/roles/destroy/tasks/parameter_store.yml +++ b/roles/destroy/tasks/parameter_store.yml @@ -2,19 +2,19 @@ set_fact: chain_env: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=region, shortnames=true, bypath=true, recursive=true ) }}" vars: - path: "/{{ prefix }}/{{ chain }}" + path: "/{{ group_names[0] }}/{{ chain }}" when: aws_access_key is defined - name: Fetch environment variables (via profile) set_fact: chain_env: "{{ lookup('aws_ssm', path, aws_profile=aws_profile, shortnames=true, bypath=true, recursive=true ) }}" vars: - path: "/{{ prefix }}/{{ chain }}" + path: "/{{ group_names[0] }}/{{ chain }}" when: aws_profile is defined - name: Remove chain variables aws_ssm_parameter_store: - name: "/{{ prefix }}/{{ chain }}/{{ item.key }}" + name: "/{{ group_names[0] }}/{{ chain }}/{{ item.key }}" value: "{{ item.value }}" state: absent profile: "{{ profile }}" diff --git a/roles/dynamodb/tasks/main.yml b/roles/dynamodb/tasks/main.yml index 9de803b..e1e50d5 100644 --- a/roles/dynamodb/tasks/main.yml +++ b/roles/dynamodb/tasks/main.yml @@ -1,13 +1,13 @@ - name: Create DynamoDB table dynamodb_table: - name: "{{ prefix }}-{{ dynamodb_table }}" + name: "{{ group_names[0] }}-{{ dynamodb_table }}" hash_key_name: LockID hash_key_type: STRING read_capacity: 1 write_capacity: 1 tags: origin: terraform - prefix: "{{ prefix }}" + prefix: "{{ group_names[0] }}" profile: "{{ profile }}" aws_access_key: "{{ access_key }}" aws_secret_key: "{{ secret_key }}" diff --git a/roles/main_infra/defaults/main.yml b/roles/main_infra/defaults/main.yml index 6edcb21..3c110e7 100644 --- a/roles/main_infra/defaults/main.yml +++ b/roles/main_infra/defaults/main.yml @@ -9,3 +9,4 @@ db_subnet_cidr: "10.0.2.0/16" dns_zone_name: "poa.internal" instance_type: "m5.large" root_block_size: 8 +db_iops: {} diff --git a/roles/main_infra/tasks/main.yml b/roles/main_infra/tasks/main.yml index a31002a..9eb203e 100644 --- a/roles/main_infra/tasks/main.yml +++ b/roles/main_infra/tasks/main.yml @@ -1,53 +1,68 @@ +- name: Ansible delete file glob + find: + paths: /tmp/ + file_type: directory + patterns: "files-{{ group_names[0] }}" + register: files_to_delete + +- name: Ansible remove file glob + file: + path: "{{ item.path }}" + state: absent + with_items: "{{ files_to_delete.files }}" + +- name: Copy files + copy: + src: "roles/main_infra/files/" + dest: "/tmp/files-{{ group_names[0] }}/" + - name: Local or remote backend selector (remote) template: src: remote-backend-selector.tf.j2 - dest: roles/main_infra/files/remote-backend-selector.tf + dest: "/tmp/files-{{ group_names[0] }}/remote-backend-selector.tf" when: - - backend|bool + - backend | bool - name: Local or remote backend selector (local) file: state: absent - dest: roles/main_infra/files/remote-backend-selector.tf + dest: "/tmp/files-{{ group_names[0] }}/remote-backend-selector.tf" when: - - backend | default('false') | bool + - not backend | default('false') | bool - name: Generating variables file template: src: terraform.tfvars.j2 - dest: roles/main_infra/files/terraform.tfvars - vars: - db_iops: "{{ db_iops | default({}) }}" + dest: "/tmp/files-{{ group_names[0] }}/terraform.tfvars" - name: Generating backend file template: src: backend.tfvars.j2 - dest: roles/main_infra/files/backend.tfvars + dest: "/tmp/files-{{ group_names[0] }}/backend.tfvars" when: backend | default('false') | bool -- name: Check if .terraform folder exists - stat: - path: "roles/main_infra/files/.terraform/" - register: stat_result - -- name: Remove .terraform folder +- name: Remove Terraform state file: - path: roles/main_infra/files/.terraform/ + path: "{{ item }}" state: absent - when: stat_result.stat.exists + with_items: + - "/tmp/files-{{ group_names[0] }}/.terraform/" + - "/tmp/files-{{ group_names[0] }}/terraform.tfstate" + - "/tmp/files-{{ group_names[0] }}/terraform.tfstate.backup" + - "/tmp/files-{{ group_names[0] }}/terraform.tfplan" - name: Generate Terraform files template: src: "{{ item.key }}" dest: "{{ item.value }}" - with_dict: { hosts.tf.j2: roles/main_infra/files/hosts.tf, routing.tf.j2: roles/main_infra/files/routing.tf, provider.tf.j2: roles/main_infra/files/provider.tf } + with_dict: { hosts.tf.j2: "/tmp/files-{{ group_names[0] }}/hosts.tf", routing.tf.j2: "/tmp/files-{{ group_names[0] }}/routing.tf", provider.tf.j2: "/tmp/files-{{ group_names[0] }}/provider.tf" } #Workaround since terraform module return unexpected error. - name: Terraform plan construct shell: "echo yes | {{ terraform_location }} {{ item }}" register: tf_plan args: - chdir: "roles/main_infra/files" + chdir: "/tmp/files-{{ group_names[0] }}" with_items: - "init{{ ' -backend-config=backend.tfvars' if backend|bool == true else '' }}" - plan -out terraform.tfplan @@ -61,35 +76,51 @@ pause: prompt: "Are you absolutely sure you want to execute the deployment plan shown above? [False]" register: user_answer - until: user_answer.user_input | lower != "false" and user_answer.user_input | lower != "no" and user_answer.user_input | lower != "true" and user_answer.user_input | lower != "yes" + until: user_answer.user_input | lower in conditional retries: 10000 delay: 1 + vars: + conditional: ['yes','no','true','false'] + when: inventory_hostname == groups['all'][0] - name: Insert vars into parameter store include: parameter_store.yml - when: user_answer.user_input | bool + when: hostvars[groups['all'][0]].user_answer.user_input | bool - name: Terraform provisioning shell: "echo yes | {{ terraform_location }} apply terraform.tfplan" args: - chdir: "roles/main_infra/files" - when: user_answer.user_input | bool + chdir: "/tmp/files-{{ group_names[0] }}" + when: hostvars[groups['all'][0]].user_answer.user_input | bool ignore_errors: True - name: Ensure Terraform resources has been provisioned shell: "echo yes | {{ terraform_location }} apply" args: - chdir: "roles/main_infra/files" - when: user_answer.user_input | bool + chdir: "/tmp/files-{{ group_names[0] }}" + when: hostvars[groups['all'][0]].user_answer.user_input | bool - name: Terraform output info into variable shell: "{{ terraform_location }} output -json" register: output args: - chdir: "roles/main_infra/files" - when: user_answer.user_input | bool + chdir: "/tmp/files-{{ group_names[0] }}" + when: hostvars[groups['all'][0]].user_answer.user_input | bool - name: Output info from Terraform debug: var: output.stdout_lines - when: user_answer.user_input | bool + when: hostvars[groups['all'][0]].user_answer.user_input | bool + +- name: Ansible delete file glob + find: + paths: /tmp/ + file_type: directory + patterns: "files-{{ group_names[0] }}" + register: files_to_delete + +- name: Ansible remove file glob + file: + path: "{{ item.path }}" + state: absent + with_items: "{{ files_to_delete.files }}" diff --git a/roles/main_infra/tasks/parameter_store.yml b/roles/main_infra/tasks/parameter_store.yml index bbf13b7..e5f9481 100644 --- a/roles/main_infra/tasks/parameter_store.yml +++ b/roles/main_infra/tasks/parameter_store.yml @@ -7,7 +7,7 @@ - name: Insert variables in PS aws_ssm_parameter_store: - name: "/{{ prefix }}/{{ chain }}/{{ item.key }}" + name: "/{{ group_names[0] }}/{{ chain }}/{{ item.key }}" value: "{{ item.value }}" profile: "{{ profile }}" aws_access_key: "{{ access_key }}" diff --git a/roles/main_infra/templates/hosts.tf.j2 b/roles/main_infra/templates/hosts.tf.j2 index 27f8d77..1b3bd7d 100644 --- a/roles/main_infra/templates/hosts.tf.j2 +++ b/roles/main_infra/templates/hosts.tf.j2 @@ -39,22 +39,22 @@ resource "aws_launch_configuration" "explorer" { } } -{% for key, value in env_vars.iteritems() %} -{% if value['USE_PLACEMENT_GROUP']|default('true') == "true" %} -resource "aws_placement_group" "explorer-{{key}}" { - name = "${var.prefix}-{{key}}-explorer-pg" +{% for key in groups[group_names[0]] %} +{% if use_placement_group | default('true') == "true" %} +resource "aws_placement_group" "explorer-{{ hostvars[key]['chain'] }}" { + name = "${var.prefix}-{{ hostvars[key]['chain'] }}-explorer-pg" strategy = "cluster" } {% endif %} {% endfor %} -{% for key, value in env_vars.iteritems() %} -resource "aws_autoscaling_group" "explorer-{{key}}" { - name = "${aws_launch_configuration.explorer.name}-asg-{{key}}" +{% for key in groups[group_names[0]] %} +resource "aws_autoscaling_group" "explorer-{{ hostvars[key]['chain'] }}" { + name = "${aws_launch_configuration.explorer.name}-asg-{{ hostvars[key]['chain'] }}" max_size = "4" min_size = "1" desired_capacity = "1" -{% if use_placement_group | default('false') == "true" %} placement_group = "${var.prefix}-{{key}}-explorer-pg" +{% if use_placement_group | default('false') == "true" %} placement_group = "${var.prefix}-{{ hostvars[key]['chain'] }}-explorer-pg" {% endif %} launch_configuration = "${aws_launch_configuration.explorer.name}" vpc_zone_identifier = ["${aws_subnet.default.id}"] @@ -92,29 +92,29 @@ resource "aws_autoscaling_group" "explorer-{{key}}" { tag { key = "chain" - value = "{{ key }}" + value = "{{ hostvars[key]['chain'] }}" propagate_at_launch = true } tag { key = "Name" - value = "{{ key }} Application" + value = "{{ hostvars[key]['chain'] }} Application" propagate_at_launch = true } } # TODO: These autoscaling policies are not currently wired up to any triggers -resource "aws_autoscaling_policy" "explorer-up" { - name = "${var.prefix}-{{key}}-explorer-autoscaling-policy-up" - autoscaling_group_name = "${aws_autoscaling_group.explorer-{{key}}.name}" +resource "aws_autoscaling_policy" "explorer-up-{{ hostvars[key]['chain'] }}" { + name = "${var.prefix}-{{ hostvars[key]['chain'] }}-explorer-autoscaling-policy-up" + autoscaling_group_name = "${aws_autoscaling_group.explorer-{{ hostvars[key]['chain'] }}.name}" adjustment_type = "ChangeInCapacity" scaling_adjustment = 1 cooldown = 300 } -resource "aws_autoscaling_policy" "explorer-down" { - name = "${var.prefix}-{{key}}-explorer-autoscaling-policy-down" - autoscaling_group_name = "${aws_autoscaling_group.explorer-{{key}}.name}" +resource "aws_autoscaling_policy" "explorer-down-{{ hostvars[key]['chain'] }}" { + name = "${var.prefix}-{{ hostvars[key]['chain'] }}-explorer-autoscaling-policy-down" + autoscaling_group_name = "${aws_autoscaling_group.explorer-{{ hostvars[key]['chain'] }}.name}" adjustment_type = "ChangeInCapacity" scaling_adjustment = -1 cooldown = 300 diff --git a/roles/main_infra/templates/routing.tf.j2 b/roles/main_infra/templates/routing.tf.j2 index 9dd190f..a6bebe6 100644 --- a/roles/main_infra/templates/routing.tf.j2 +++ b/roles/main_infra/templates/routing.tf.j2 @@ -58,12 +58,12 @@ resource "aws_lb_target_group" "explorer" { } } -{% for key, value in env_vars.iteritems() %} +{% for host in groups[group_names[0]] %} resource "aws_alb_listener" "alb_listener{{loop.index-1}}" { load_balancer_arn = "${aws_lb.explorer.*.arn[{{loop.index-1}}]}" port = "${lookup(var.use_ssl,element(var.chains,{{loop.index-1}})) ? "443" : "80" }" protocol = "${lookup(var.use_ssl,element(var.chains,{{loop.index-1}})) ? "HTTPS" : "HTTP" }" -{% if value['ECTO_USE_SSL']|default('false') == "true" %} +{% if hostvars[host]['env_vars']['ECTO_USE_SSL']|default('false') == "true" %} ssl_policy = "${lookup(var.alb_ssl_policy,element(var.chains,{{loop.index-1}}))}" certificate_arn = "${lookup(var.alb_certificate_arn,element(var.chains,{{loop.index-1}}))}" {% endif %} diff --git a/roles/main_infra/templates/terraform.tfvars.j2 b/roles/main_infra/templates/terraform.tfvars.j2 index 0ca8988..1a08696 100644 --- a/roles/main_infra/templates/terraform.tfvars.j2 +++ b/roles/main_infra/templates/terraform.tfvars.j2 @@ -45,55 +45,55 @@ chains = [ {% endfor %} ] -db_id = { +chain_db_id = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_id'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_name = { +chain_db_name = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_name'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_username = { +chain_db_username = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_username'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_password = { +chain_db_password = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_password'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_instance_class = { +chain_db_instance_class = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_instance_class'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_storage = { +chain_db_storage = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_storage'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_storage_type = { +chain_db_storage_type = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_storage_type'] }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_iops = { +chain_db_iops = { {% for host in groups[group_names[0]] %} -{{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_iops'] }}"{% if not loop.last %},{% endif %} +{{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_iops']|default('0') }}"{% if not loop.last %},{% endif %} {% endfor %} } -db_version = { +chain_db_version = { {% for host in groups[group_names[0]] %} {{ hostvars[host]['chain'] }} = "{{ hostvars[host]['db_version'] }}"{% if not loop.last %},{% endif %} {% endfor %} diff --git a/roles/main_software/tasks/main.yml b/roles/main_software/tasks/main.yml index 5e8b762..08b50bc 100644 --- a/roles/main_software/tasks/main.yml +++ b/roles/main_software/tasks/main.yml @@ -1,7 +1,7 @@ - name: Clone BlockScout git: repo: "{{ blockscout_repo }}" - dest: "blockscout-{{ group_names[0] }}-{{ chain }}" + dest: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}" version: "{{ branch }}" force: true when: skip_fetch | bool != true @@ -9,75 +9,42 @@ - name: Git clean command: "git clean -fdx" args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}" + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}" when: skip_fetch | bool != true - name: Merge branches command: "git merge {{ merge_commit_item }}" args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}" + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}" when: merge_commit_item and not skip_fetch | bool vars: merge_commit_item: "{{ merge_commit | default(false) }}" - name: Copy web config files copy: - src: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/config/dev.secret.exs.example" - dest: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/config/dev.secret.exs" + src: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/config/dev.secret.exs.example" + dest: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/config/prod.secret.exs" - name: Template explorer config files template: src: dev.secret.exs.j2 - dest: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/config/dev.secret.exs" + dest: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/config/prod.secret.exs" when: ps_user is defined - name: Copy default explorer config files copy: - src: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/config/dev.secret.exs.example" - dest: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/config/dev.secret.exs" + src: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/config/dev.secret.exs.example" + dest: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/config/prod.secret.exs" when: ps_user is undefined or ps_user == "" - name: Remove static assets from previous deployment, if any file: - path: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/priv/static" + path: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/priv/static" state: absent -- name: Compile BlockScout - command: "mix do {{ item }}" - args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}" - with_items: - - deps.get - - local.rebar --force - - deps.compile - - compile - - ecto.drop - - ecto.create - - ecto.migrate - -- name: Install Node modules at apps/block_scout_web/assets - command: npm install - args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/assets" - -- name: Execute webpack.js at apps/block_scout_web/assets/node_modules/webpack/bin - command: node_modules/webpack/bin/webpack.js --mode production - args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/assets" - -- name: Instal Node modules at apps/explorer - command: npm install - args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer" - -- name: Install SSL certificates - command: mix phx.gen.cert blockscout blockscout.local - args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web" - - name: Fetch environment variables (via access key) set_fact: - env: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=aws_region|default('us-east-1'), shortnames=true, bypath=true, recursive=true ) }}" + env_compiled: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=aws_region|default('us-east-1'), shortnames=true, bypath=true, recursive=true ) }}" vars: path: "/{{ group_names[0] }}/{{ chain }}" when: aws_access_key is defined @@ -100,7 +67,7 @@ - name: Override env variables set_fact: - env_compiled: "{{ env_compilated | combine(lower_env) }}" + env_compiled: "{{ env_compiled | combine(lower_env) }}" when: lower_env is defined - name: Uppercase chain @@ -110,19 +77,59 @@ vars: upper_env: {} +- name: Add server port + set_fact: + server_port: "{{ 65535|random(seed=inventory_hostname,start=1024) }}" + +- name: Combine server env + set_fact: + server_env: "{{ upper_env | combine({'NETWORK_PATH':'/','PORT':server_port,'MIX_ENV':'prod','DATABASE_URL':'postgresql://' ~ ps_user ~ ':' ~ ps_password ~ '@' ~ ps_host ~ ':5432/' ~ ps_db}) }}" + +- name: Compile BlockScout + command: "mix do {{ item }}" + args: + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}" + environment: "{{ server_env }}" + with_items: + - deps.get + - local.rebar --force + - deps.compile + - compile + - ecto.drop + - ecto.create + - ecto.migrate + +- name: Install Node modules at apps/block_scout_web/assets + environment: "{{ server_env }}" + command: npm install + args: + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/assets" + +- name: Execute webpack.js at apps/block_scout_web/assets/node_modules/webpack/bin + environment: "{{ server_env }}" + command: node_modules/webpack/bin/webpack.js --mode production + args: + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/assets" + +- name: Instal Node modules at apps/explorer + environment: "{{ server_env }}" + command: npm install + args: + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer" + +- name: Install SSL certificates + environment: "{{ server_env }}" + command: mix phx.gen.cert blockscout blockscout.local + args: + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web" + - name: Start server block: - - set_fact: - server_port: "{{ 65535|random(seed=inventory_hostname,start=1024) }}" - - - set_fact: - server_env: "{{ upper_env | combine({'NETWORK_PATH':'/','PORT':server_port,'MIX_ENV':'prod'}) }}" - - name: Start server command: "mix phx.server" environment: "{{ server_env }}" args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}" + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}" async: 10000 poll: 0 @@ -152,38 +159,42 @@ when: prompt is failed - name: Build static assets + environment: "{{ server_env }}" command: mix phx.digest args: - chdir: "blockscout-{{ group_names[0] }}-{{ chain }}" + chdir: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}" - name: User prompt pause: prompt: "Would you like to remove staging dependencies? [Yes/No] Default: Yes" register: user_answer - until: user_answer.user_input|lower != "false" and user_answer.user_input|lower != "no" and user_answer.user_input|lower != "true" and user_answer.user_input|lower != "yes" + until: user_answer.user_input | lower in conditional retries: 10000 delay: 1 + vars: + conditional: ['yes','no','true','false'] + when: inventory_hostname == groups['all'][0] - name: Remove dev dependencies file: state: absent path: "{{ item }}" with_items: - - "blockscout-{{ group_names[0] }}-{{ chain }}/_build/" - - "blockscout-{{ group_names[0] }}-{{ chain }}/deps/" - - "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/assets/node_modules/" - - "blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/node_modules/" - - "blockscout-{{ group_names[0] }}-{{ chain }}/logs/dev/" - when: user_answer.user_input | lower | bool + - "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/_build/" + - "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/deps/" + - "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/assets/node_modules/" + - "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/explorer/node_modules/" + - "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/logs/dev/" + when: hostvars[groups['all'][0]].user_answer.user_input | lower | bool - name: Fix bug with favicon replace: regexp: '\"favicon\.ico\"\:\"favicon-[a-z0-9]+?\.ico\"' replace: '"images/favicon.ico":"favicon.ico"' - path: "blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/priv/static/cache_manifest.json" + path: "/tmp/blockscout-{{ group_names[0] }}-{{ chain }}/apps/block_scout_web/priv/static/cache_manifest.json" - name: Upload Blockscout to S3 - command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} aws deploy push --application-name={{ group_names[0] }}-explorer --s3-location s3://{{ group_names[0] }}-explorer-codedeploy-releases/blockscout-{{ group_names[0] }}-{{ chain }}.zip --source=blockscout-{{ group_names[0] }}-{{ chain }} {{ '--profile='~aws_profile if aws_profile is defined else '' }}" + command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} aws deploy push --application-name={{ group_names[0] }}-explorer --s3-location s3://{{ group_names[0] }}-explorer-codedeploy-releases/blockscout-{{ group_names[0] }}-{{ chain }}.zip --source=/tmp/blockscout-{{ group_names[0] }}-{{ chain }} {{ '--profile='~aws_profile if aws_profile is defined else '' }}" register: push_output - name: Upload output @@ -194,9 +205,12 @@ pause: prompt: "Do you want to update the Parameter Store variables? [Yes/No] Default: Yes" register: user_answer - until: user_answer.user_input | lower != "false" and user_answer.user_input | lower != "no" and user_answer.user_input | lower != "true" and user_answer.user_input | lower != "yes" + until: user_answer.user_input | lower in conditional retries: 10000 delay: 1 + vars: + conditional: ['yes','no','true','false'] + when: inventory_hostname == groups['all'][0] - name: Update chain variables aws_ssm_parameter_store: @@ -212,16 +226,19 @@ profile: "{{ aws_profile|default(omit) }}" region: "{{ aws_region|default(omit) }}" with_dict: "{{ lower_env }}" - when: user_answer.user_input | lower | bool + when: hostvars[groups['all'][0]].user_answer.user_input | lower | bool - name: User prompt pause: prompt: "Do you want to deploy BlockScout? [Yes/No] Default: Yes" register: user_answer - until: user_answer.user_input | lower != "false" and user_answer.user_input | lower != "no" and user_answer.user_input | lower != "true" and user_answer.user_input | lower != "yes" + until: user_answer.user_input | lower in conditional retries: 10000 delay: 1 + vars: + conditional: ['yes','no','true','false'] + when: inventory_hostname == groups['all'][0] - name: Deploy Blockscout - command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} {{ push_output.stdout_lines[1] }} --deployment-group-name {{ group_names[0] }}-explorer-dg{{ groups[group_names[0]].index(inventory_hostname) }} --deployment-config-name CodeDeployDefault.OneAtATime --description '{{ env_compiled['BLOCKSCOUT_VERSION'] }}' {{ '--profile='~aws_profile if aws_profile is defined else '' }}" - when: user_answer.user_input | lower | bool + command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} {{ push_output.stdout_lines[1] }} --deployment-group-name {{ group_names[0] }}-explorer-dg{{ groups[group_names[0]].index(inventory_hostname) }} --deployment-config-name CodeDeployDefault.OneAtATime {{ '--profile='~aws_profile if aws_profile is defined else '' }}" + when: hostvars[groups['all'][0]].user_answer.user_input | lower | bool diff --git a/roles/s3_config/tasks/main.yml b/roles/s3_config/tasks/main.yml index 14a0602..3cb963f 100644 --- a/roles/s3_config/tasks/main.yml +++ b/roles/s3_config/tasks/main.yml @@ -1,5 +1,5 @@ - name: "Loop over config files" - include: subtasks.yml file={{item}} + include: config.yml file={{item}} with_items: - "group_vars/all.yml" - "group_vars/{{ group_names[0] }}"