From 66a4ff935635a55c53987ab27e9da66cd634378a Mon Sep 17 00:00:00 2001 From: "a@a.ru" Date: Wed, 24 Apr 2019 03:02:33 +0300 Subject: [PATCH 1/6] Several user improvements --- .gitignore | 1 + attach_existing_rds.yml | 4 -- deploy_infra.yml | 4 -- deploy_software.yml | 4 -- destroy.yml | 4 -- group_vars/all.yml.example | 17 +++---- group_vars/infrastructure.yml.example | 3 -- roles/attach_existing_rds/tasks/main.yml | 8 +++ roles/destroy/tasks/main.yml | 33 ++++++++++++ roles/destroy/tasks/parameter_store.yml | 29 +++++++++++ roles/dynamodb/tasks/main.yml | 9 ++++ roles/main_infra/files/provider.tf | 5 -- roles/main_infra/files/variables.tf | 4 +- roles/main_infra/tasks/main.yml | 2 +- roles/main_infra/tasks/parameter_store.yml | 13 +++-- roles/main_infra/templates/backend.tfvars.j2 | 1 - roles/main_infra/templates/hosts.tf.j2 | 2 + roles/main_infra/templates/provider.tf.j2 | 7 +++ .../templates/remote-backend-selector.tf.j2 | 7 +++ .../main_infra/templates/terraform.tfvars.j2 | 25 +++++---- roles/main_software/tasks/main.yml | 51 ++++++++++++------- roles/s3/tasks/main.yml | 36 +++++++++++++ 22 files changed, 199 insertions(+), 70 deletions(-) create mode 100644 roles/destroy/tasks/parameter_store.yml delete mode 100644 roles/main_infra/files/provider.tf create mode 100644 roles/main_infra/templates/provider.tf.j2 diff --git a/.gitignore b/.gitignore index 003ecba..236bbb8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ roles/main_infra/files/remote-backend-selector.tf roles/main_infra/files/terraform.tfvars roles/main_infra/files/hosts.tf roles/main_infra/files/routing.tf +roles/main_infra/files/provider.tf *.backup # Sensitive information diff --git a/attach_existing_rds.yml b/attach_existing_rds.yml index f8a2aed..6dfe35e 100644 --- a/attach_existing_rds.yml +++ b/attach_existing_rds.yml @@ -9,7 +9,3 @@ - name: "confirmation" prompt: "Are you sure you want to attach the existing RDS? If backend variable is set to True, this action includes creating the S3 and DynamoDB table for storing Terraform state files." default: False - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" diff --git a/deploy_infra.yml b/deploy_infra.yml index 3f2ec16..c85c19c 100644 --- a/deploy_infra.yml +++ b/deploy_infra.yml @@ -5,7 +5,3 @@ - { role: s3, when: "backend|bool == true" } - { role: dynamodb, when: "backend|bool == true" } - { role: main_infra } - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" diff --git a/deploy_software.yml b/deploy_software.yml index 72aac83..acfa6d4 100644 --- a/deploy_software.yml +++ b/deploy_software.yml @@ -2,10 +2,6 @@ hosts: localhost roles: - { role: s3, when: "backend|bool == true" } - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" - name: Deploy BlockScout hosts: localhost diff --git a/destroy.yml b/destroy.yml index 0d12950..e39b036 100644 --- a/destroy.yml +++ b/destroy.yml @@ -6,7 +6,3 @@ - name: "confirmation" prompt: "Are you sure you want to destroy all the infra?" default: False - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" diff --git a/group_vars/all.yml.example b/group_vars/all.yml.example index 08c3a27..ae7323d 100644 --- a/group_vars/all.yml.example +++ b/group_vars/all.yml.example @@ -3,9 +3,11 @@ ansible_python_interpreter: "/usr/bin/python3" # Common variables -## Credentials to connect to AWS -aws_access_key: "" -aws_secret_key: "" +## Credentials to connect to AWS. Either keypair or CLI profile name should be specified. If nothing is specified, the default AWS keypair is used. Region must be specified in all the cases. +#aws_access_key: "" +#aws_secret_key: "" +#aws_profile: "" +aws_region: "us-east-1" ## If set to true backend will be uploaded and stored at S3 bucket, so you can easily manage your deployment from any machine. It is highly recommended to do not change this variable backend: true @@ -61,9 +63,6 @@ chain_custom_environment: ADDRESS_WITH_BALANCES_UPDATE_INTERVAL: 1800 #Interval in seconds to restart the task, which calculates addresses with balances LINK_TO_OTHER_EXPLORERS: "false" # If true, links to other explorers are added in the footer USE_PLACEMENT_GROUP: "false" # If true, BlockScout instance will be created in the placement group - EXQ_BLOCKS_CONCURRENCY="1" - EXQ_INTERNAL_TRANSACTIONS_CONCURRENCY="1" - EXQ_TRANSACTIONS_CONCURRENCY="1" - -## Region. It is recommended to deploy to us-east-1 as some of the other regions fails due to varied reasons -region: "us-east-1" + EXQ_BLOCKS_CONCURRENCY: "1" + EXQ_INTERNAL_TRANSACTIONS_CONCURRENCY: "1" + EXQ_TRANSACTIONS_CONCURRENCY: "1" diff --git a/group_vars/infrastructure.yml.example b/group_vars/infrastructure.yml.example index feb31b4..6532971 100644 --- a/group_vars/infrastructure.yml.example +++ b/group_vars/infrastructure.yml.example @@ -24,9 +24,6 @@ dns_zone_name: "poa.internal" ## Size of the EC2 instance EBS root volume root_block_size: 120 -## The following variable represents the elixir version that will be used to run BlockScout instance. Will be used to download the Elixir at the following link: https://github.com/elixir-lang/elixir/releases/download/{{ elixir_version }}/Precompiled.zip -elixir_version: "v1.7.4" - # DB related variables ## This value represents the name of the DB that will be created/attached. Must be unique. Will be prefixed with `prefix` variable. diff --git a/roles/attach_existing_rds/tasks/main.yml b/roles/attach_existing_rds/tasks/main.yml index 8f018dd..d094e13 100644 --- a/roles/attach_existing_rds/tasks/main.yml +++ b/roles/attach_existing_rds/tasks/main.yml @@ -16,6 +16,8 @@ template: src: roles/main_infra/templates/terraform.tfvars.j2 dest: roles/main_infra/files/terraform.tfvars + vars: + db_iops: "{{ chain_db_iops | default({}) }}" - name: Generating backend file template: @@ -23,6 +25,12 @@ dest: roles/main_infra/files/backend.tfvars when: backend|bool == true +- name: Generate Terraform files + template: + src: "{{ item.key }}" + dest: "{{ item.value }}" + with_dict: {roles/main_infra/templates/hosts.tf.j2: roles/main_infra/files/hosts.tf,roles/main_infra/templates/routing.tf.j2: roles/main_infra/files/routing.tf,roles/main_infra/templates/provider.tf.j2: roles/main_infra/files/provider.tf} + #Workaround since terraform module return unexpected error. - name: Initialize Terraform shell: "echo yes | {{ terraform_location }} init{{ ' -backend-config=backend.tfvars' if backend|bool == true else '' }}" diff --git a/roles/destroy/tasks/main.yml b/roles/destroy/tasks/main.yml index feab63f..e320ab6 100644 --- a/roles/destroy/tasks/main.yml +++ b/roles/destroy/tasks/main.yml @@ -16,6 +16,8 @@ template: src: roles/main_infra/templates/terraform.tfvars.j2 dest: roles/main_infra/files/terraform.tfvars + vars: + db_iops: "{{ chain_db_iops | default({}) }}" - name: Generating backend file template: @@ -23,6 +25,12 @@ dest: roles/main_infra/files/backend.tfvars when: backend|bool == true +- name: Generate Terraform files + template: + src: "{{ item.key }}" + dest: "{{ item.value }}" + with_dict: {roles/main_infra/templates/hosts.tf.j2: roles/main_infra/files/hosts.tf,roles/main_infra/templates/routing.tf.j2: roles/main_infra/files/routing.tf,roles/main_infra/templates/provider.tf.j2: roles/main_infra/files/provider.tf} + # This is due to the TF0.11 bug which do not allow to completely destroy resources if interpolation syntax is used in outputs.tf at edge cases - name: Check if outputs.tf exists stat: path=roles/main_infra/files/outputs.tf @@ -40,6 +48,13 @@ - "init {{ '-backend-config=backend.tfvars' if backend|bool == true else '' }}" - destroy +- name: Delete vars from parameter store + include: parameter_store.yml + loop: "{{ chain_custom_environment.keys() }}" + loop_control: + loop_var: chain + index_var: index + - name: Check if outputs.tf.backup exists stat: path=roles/main_infra/files/outputs.tf.backup register: outputs_backup_stat @@ -58,9 +73,27 @@ name: "{{ prefix }}-{{ bucket }}" state: absent force: yes + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" when: user_answer.user_input|bool == True - dynamodb_table: name: "{{ prefix }}-{{ dynamodb_table }}" state: absent + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" when: user_answer.user_input|bool == True diff --git a/roles/destroy/tasks/parameter_store.yml b/roles/destroy/tasks/parameter_store.yml new file mode 100644 index 0000000..5c7ec27 --- /dev/null +++ b/roles/destroy/tasks/parameter_store.yml @@ -0,0 +1,29 @@ +- name: Fetch environment variables (via access key) + set_fact: + chain_env: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=region, shortnames=true, bypath=true, recursive=true ) }}" + vars: + path: "/{{ prefix }}/{{ chain }}" + when: aws_access_key is defined + +- name: Fetch environment variables (via profile) + set_fact: + chain_env: "{{ lookup('aws_ssm', path, aws_profile=aws_profile, shortnames=true, bypath=true, recursive=true ) }}" + vars: + path: "/{{ prefix }}/{{ chain }}" + when: aws_profile is defined + +- name: Remove chain variables + aws_ssm_parameter_store: + name: "/{{ prefix }}/{{ chain }}/{{ item.key }}" + value: "{{ item.value }}" + state: absent + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" + with_dict: "{{ chain_env }}" diff --git a/roles/dynamodb/tasks/main.yml b/roles/dynamodb/tasks/main.yml index c4324e1..9de803b 100644 --- a/roles/dynamodb/tasks/main.yml +++ b/roles/dynamodb/tasks/main.yml @@ -8,3 +8,12 @@ tags: origin: terraform prefix: "{{ prefix }}" + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" diff --git a/roles/main_infra/files/provider.tf b/roles/main_infra/files/provider.tf deleted file mode 100644 index 761911c..0000000 --- a/roles/main_infra/files/provider.tf +++ /dev/null @@ -1,5 +0,0 @@ -provider "aws" { - version = "~> 1.15" - - region = "${var.region}" -} diff --git a/roles/main_infra/files/variables.tf b/roles/main_infra/files/variables.tf index e0199ab..482bc19 100644 --- a/roles/main_infra/files/variables.tf +++ b/roles/main_infra/files/variables.tf @@ -7,13 +7,15 @@ variable "db_subnet_cidr" {} variable "dns_zone_name" {} variable "instance_type" {} variable "root_block_size" {} + variable "pool_size" { default = {} } -variable "elixir_version" {} + variable "use_placement_group" { default = {} } + variable "key_content" { default = "" } diff --git a/roles/main_infra/tasks/main.yml b/roles/main_infra/tasks/main.yml index 5d24346..e4d9526 100644 --- a/roles/main_infra/tasks/main.yml +++ b/roles/main_infra/tasks/main.yml @@ -40,7 +40,7 @@ template: src: "{{ item.key }}" dest: "{{ item.value }}" - with_dict: {hosts.tf.j2: roles/main_infra/files/hosts.tf,routing.tf.j2: roles/main_infra/files/routing.tf} + with_dict: {hosts.tf.j2: roles/main_infra/files/hosts.tf,routing.tf.j2: roles/main_infra/files/routing.tf,provider.tf.j2: roles/main_infra/files/provider.tf} #Workaround since terraform module return unexpected error. - name: Terraform plan construct diff --git a/roles/main_infra/tasks/parameter_store.yml b/roles/main_infra/tasks/parameter_store.yml index b7e2dcc..03edc30 100644 --- a/roles/main_infra/tasks/parameter_store.yml +++ b/roles/main_infra/tasks/parameter_store.yml @@ -9,8 +9,13 @@ aws_ssm_parameter_store: name: "/{{ prefix }}/{{ chain }}/{{ item.key }}" value: "{{ item.value }}" + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" with_dict: "{{ chain_ps_env }}" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" diff --git a/roles/main_infra/templates/backend.tfvars.j2 b/roles/main_infra/templates/backend.tfvars.j2 index 5a3905d..c086578 100644 --- a/roles/main_infra/templates/backend.tfvars.j2 +++ b/roles/main_infra/templates/backend.tfvars.j2 @@ -1,4 +1,3 @@ -region = "{{ ansible_env.AWS_REGION }}" bucket = "{{ prefix }}-{{ bucket }}" dynamodb_table = "{{ prefix }}-{{ dynamodb_table }}" key = "terraform.tfstate" diff --git a/roles/main_infra/templates/hosts.tf.j2 b/roles/main_infra/templates/hosts.tf.j2 index 3d2e515..6bf080f 100644 --- a/roles/main_infra/templates/hosts.tf.j2 +++ b/roles/main_infra/templates/hosts.tf.j2 @@ -40,10 +40,12 @@ resource "aws_launch_configuration" "explorer" { } {% for key, value in chain_custom_environment.iteritems() %} +{% if value['USE_PLACEMENT_GROUP'] == "true" %} resource "aws_placement_group" "explorer-{{key}}" { name = "${var.prefix}-{{key}}-explorer-pg" strategy = "cluster" } +{% endif %} {% endfor %} {% for key, value in chain_custom_environment.iteritems() %} diff --git a/roles/main_infra/templates/provider.tf.j2 b/roles/main_infra/templates/provider.tf.j2 new file mode 100644 index 0000000..76bed10 --- /dev/null +++ b/roles/main_infra/templates/provider.tf.j2 @@ -0,0 +1,7 @@ +provider "aws" { + version = "~> 1.15" +{% if aws_access_key is undefined %} + profile = "{{ aws_profile|default("default") }}" +{% endif %} + region = "{{ aws_region|default("us-east-1") }}" +} diff --git a/roles/main_infra/templates/remote-backend-selector.tf.j2 b/roles/main_infra/templates/remote-backend-selector.tf.j2 index cde57e0..56f0b2a 100644 --- a/roles/main_infra/templates/remote-backend-selector.tf.j2 +++ b/roles/main_infra/templates/remote-backend-selector.tf.j2 @@ -1,4 +1,11 @@ terraform { backend "s3" { + {% if aws_access_key is undefined %} + profile = "{{ aws_profile|default("default") }}" + {% else %} + access_key = "{{ aws_access_key }}" + secret_key = "{{ aws_secret_key }}" + {% endif %} + region = "{{ aws_region }}" } } diff --git a/roles/main_infra/templates/terraform.tfvars.j2 b/roles/main_infra/templates/terraform.tfvars.j2 index f1fada9..f95c72d 100644 --- a/roles/main_infra/templates/terraform.tfvars.j2 +++ b/roles/main_infra/templates/terraform.tfvars.j2 @@ -1,14 +1,12 @@ -region = "{{ ansible_env.AWS_REGION }}" -prefix = "{{ prefix }}" -key_name = "{{ ec2_ssh_key_name }}" -key_content = "{{ ec2_ssh_key_content }}" -vpc_cidr = "{{ vpc_cidr }}" -public_subnet_cidr = "{{ public_subnet_cidr }}" -db_subnet_cidr = "{{ db_subnet_cidr }}" -dns_zone_name = "{{ dns_zone_name }}" -instance_type = "{{ instance_type }}" -root_block_size = "{{ root_block_size }}" -elixir_version = "{{ elixir_version }}" +prefix = "{{ prefix }}" +key_name = "{{ ec2_ssh_key_name }}" +key_content = "{{ ec2_ssh_key_content }}" +vpc_cidr = "{{ vpc_cidr }}" +public_subnet_cidr = "{{ public_subnet_cidr }}" +db_subnet_cidr = "{{ db_subnet_cidr }}" +dns_zone_name = "{{ dns_zone_name }}" +instance_type = "{{ instance_type }}" +root_block_size = "{{ root_block_size }}" pool_size = { {% for key, value in chain_custom_environment.iteritems() %} @@ -28,15 +26,16 @@ use_ssl = { {% endfor %} } + alb_ssl_policy = { {% for key, value in chain_custom_environment.iteritems() %} -{{ key }}="{{ value['ALB_SSL_POLICY'] }}"{% if not loop.last %},{% endif %} +{{ key }}="{{ value['ALB_SSL_POLICY']|default('') }}"{% if not loop.last %},{% endif %} {% endfor %} } alb_certificate_arn = { {% for key, value in chain_custom_environment.iteritems() %} -{{ key }}="{{ value['ALB_CERTIFICATE_ARN'] }}"{% if not loop.last %},{% endif %} +{{ key }}="{{ value['ALB_CERTIFICATE_ARN']|default('') }}"{% if not loop.last %},{% endif %} {% endfor %} } diff --git a/roles/main_software/tasks/main.yml b/roles/main_software/tasks/main.yml index 8dbc9f0..133d257 100644 --- a/roles/main_software/tasks/main.yml +++ b/roles/main_software/tasks/main.yml @@ -30,6 +30,13 @@ template: src: dev.secret.exs.j2 dest: "blockscout-{{ chain }}/apps/explorer/config/dev.secret.exs" + when: ps_db is defined + +- name: Copy default explorer config files + copy: + src: "blockscout-{{ chain }}/apps/explorer/config/dev.secret.exs.example" + dest: "blockscout-{{ chain }}/apps/explorer/config/dev.secret.exs" + when: ps_db is undefined or ps_db == "" - name: Remove static assets from previous deployment, if any file: @@ -69,24 +76,34 @@ args: chdir: "blockscout-{{ chain }}/apps/block_scout_web" -- name: Fetch environment variables +- name: Fetch environment variables (via access key) set_fact: chain_env: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=region, shortnames=true, bypath=true, recursive=true ) }}" vars: path: "/{{ prefix }}/{{ chain }}" + when: aws_access_key is defined + +- name: Fetch environment variables (via profile) + set_fact: + chain_env: "{{ lookup('aws_ssm', path, aws_profile=aws_profile, shortnames=true, bypath=true, recursive=true ) }}" + vars: + path: "/{{ prefix }}/{{ chain }}" + when: aws_profile is defined - name: Make config variables lowercase set_fact: chain_lower_env: "{{ chain_lower_env | combine ({item.key|lower : item.value}) }}" with_dict: "{{ chain_custom_environment_chain }}" + when: chain_custom_environment_chain|length > 0 vars: chain_lower_env: {} - chain_custom_environment_chain: "{{ chain_cec[chain] | default({}) }}" + chain_custom_environment_chain: "{{ chain_cec[chain] | default({}) if chain_cec[chain]>0 else {} }}" chain_cec: "{{ chain_custom_environment | default ({}) }}" - name: Override env variables set_fact: chain_env: "{{ chain_env | combine(chain_lower_env) }}" + when: chain_lower_env is defined - name: Uppercase chain set_fact: @@ -115,11 +132,13 @@ msg: "Execution aborted." always: - name: kill server - command: "pkill -e {{ item }}" + command: "pkill -f {{ item }}" with_items: - beam.smp - node + - erlang failed_when: false + when: - name: Build static assets command: mix phx.digest @@ -151,12 +170,8 @@ - "blockscout-{{ chain }}/apps/block_scout_web/priv/static/favicon*" - name: Upload Blockscout to S3 - command: "aws deploy push --application-name={{ prefix }}-explorer --s3-location s3://{{ prefix }}-explorer-codedeploy-releases/blockscout-{{ chain }}.zip --source=blockscout-{{ chain }}" + command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} aws deploy push --application-name={{ prefix }}-explorer --s3-location s3://{{ prefix }}-explorer-codedeploy-releases/blockscout-{{ chain }}.zip --source=blockscout-{{ chain }} {{ '--profile='~aws_profile if aws_profile is defined else '' }}" register: push_output - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" - name: Upload output debug: @@ -171,11 +186,17 @@ aws_ssm_parameter_store: name: "/{{ prefix }}/{{ chain }}/{{ item.key }}" value: "{{ item.value }}" + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" with_dict: "{{ chain_env }}" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" + when: user_answer.user_input|lower != "false" and user_answer.user_input|lower != "no" - name: User prompt @@ -184,9 +205,5 @@ register: user_answer - name: Deploy Blockscout - command: "{{ push_output.stdout_lines[1] }} --deployment-group-name {{ prefix }}-explorer-dg{{ index }} --deployment-config-name CodeDeployDefault.OneAtATime --description '{{ chain_upper_env['BLOCKSCOUT_VERSION'] }}'" + command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} {{ push_output.stdout_lines[1] }} --deployment-group-name {{ prefix }}-explorer-dg{{ index }} --deployment-config-name CodeDeployDefault.OneAtATime --description '{{ chain_upper_env['BLOCKSCOUT_VERSION'] }}' {{ '--profile='~aws_profile if aws_profile is defined else '' }}" when: user_answer.user_input|lower != "false" and user_answer.user_input|lower != "no" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_REGION: "{{ region }}" diff --git a/roles/s3/tasks/main.yml b/roles/s3/tasks/main.yml index f706a9d..0d666ff 100644 --- a/roles/s3/tasks/main.yml +++ b/roles/s3/tasks/main.yml @@ -3,6 +3,15 @@ bucket: "{{ prefix }}-{{ bucket }}" mode: create permission: private + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" - name: Apply tags and versioning to create S3 bucket s3_bucket: @@ -11,6 +20,15 @@ tags: origin: terraform prefix: "{{ prefix }}" + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" - name: Add lifecycle management policy to created S3 bucket s3_lifecycle: @@ -19,6 +37,15 @@ noncurrent_version_expiration_days: 90 status: enabled state: present + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" - name: Check if config file exists stat: @@ -50,6 +77,15 @@ object: all.yml src: "{{ playbook_dir }}/group_vars/all.yml.temp" mode: put + profile: "{{ profile }}" + aws_access_key: "{{ access_key }}" + aws_secret_key: "{{ secret_key }}" + region: "{{ region }}" + vars: + access_key: "{{ aws_access_key|default(omit) }}" + secret_key: "{{ aws_secret_key|default(omit) }}" + profile: "{{ aws_profile|default(omit) }}" + region: "{{ aws_region|default(omit) }}" when: upload_config_to_s3|bool == True and stat_result.stat.exists == True - name: Remove temp file From 74c2eefb191c537b3222702f23e3b8e61e23e954 Mon Sep 17 00:00:00 2001 From: "a@a.ru" Date: Wed, 24 Apr 2019 12:42:04 +0300 Subject: [PATCH 2/6] Remove obsolete files and vars --- Makefile | 33 ----------------------------- appspec.yml.example | 19 ----------------- roles/main_infra/files/variables.tf | 1 - 3 files changed, 53 deletions(-) delete mode 100644 Makefile delete mode 100644 appspec.yml.example diff --git a/Makefile b/Makefile deleted file mode 100644 index 49f2010..0000000 --- a/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -.PHONY: help - -IMAGE_NAME ?= poa-aws -INFRA_PREFIX ?= poa-example -KEY_PAIR ?= poa - -help: - @echo "$(IMAGE_NAME)" - @perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -check: lint ## Run linters and validation - @bin/infra precheck - @terraform validate -var-file=ignore.tfvars base - @if [ -f main.tfvars ]; then \ - terraform validate \ - -var='db_password=foo' \ - -var='new_relic_app_name=foo' \ - -var='new_relic_license_key=foo' \ - -var-file=main.tfvars main; \ - fi - @rm ignore.tfvars - -format: ## Apply canonical formatting to Terraform files - @terraform fmt - -lint: shellcheck check-format ## Lint scripts and config files - -check-format: - @terraform fmt -check=true - -shellcheck: - @shellcheck --shell=bash bin/infra - @shellcheck --shell=bash modules/stack/libexec/init.sh diff --git a/appspec.yml.example b/appspec.yml.example deleted file mode 100644 index ee2a8f0..0000000 --- a/appspec.yml.example +++ /dev/null @@ -1,19 +0,0 @@ -version: 0.0 -os: linux -files: - - source: . - destination: /opt/app -hooks: - ApplicationStop: - - location: bin/stop.sh - timeout: 300 - AfterInstall: - - location: bin/build.sh - ApplicationStart: - - location: bin/migrate.sh - timeout: 300 - - location: bin/start.sh - timeout: 3600 - ValidateService: - - location: bin/health_check.sh - timeout: 3600 diff --git a/roles/main_infra/files/variables.tf b/roles/main_infra/files/variables.tf index 482bc19..d9b97a5 100644 --- a/roles/main_infra/files/variables.tf +++ b/roles/main_infra/files/variables.tf @@ -1,4 +1,3 @@ -variable "region" {} variable "prefix" {} variable "key_name" {} variable "vpc_cidr" {} From f834c93eb6fb1b944124b6f92666888b86b72fc5 Mon Sep 17 00:00:00 2001 From: "a@a.ru" Date: Wed, 24 Apr 2019 14:30:29 +0300 Subject: [PATCH 3/6] update defaults --- roles/main_infra/templates/hosts.tf.j2 | 4 ++-- roles/main_infra/templates/routing.tf.j2 | 2 +- roles/main_infra/templates/terraform.tfvars.j2 | 6 +++--- roles/main_software/tasks/main.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/main_infra/templates/hosts.tf.j2 b/roles/main_infra/templates/hosts.tf.j2 index 6bf080f..666ef3f 100644 --- a/roles/main_infra/templates/hosts.tf.j2 +++ b/roles/main_infra/templates/hosts.tf.j2 @@ -40,7 +40,7 @@ resource "aws_launch_configuration" "explorer" { } {% for key, value in chain_custom_environment.iteritems() %} -{% if value['USE_PLACEMENT_GROUP'] == "true" %} +{% if value['USE_PLACEMENT_GROUP']|default('true') == "true" %} resource "aws_placement_group" "explorer-{{key}}" { name = "${var.prefix}-{{key}}-explorer-pg" strategy = "cluster" @@ -54,7 +54,7 @@ resource "aws_autoscaling_group" "explorer-{{key}}" { max_size = "4" min_size = "1" desired_capacity = "1" -{% if value['USE_PLACEMENT_GROUP'] == "true" %} placement_group = "${var.prefix}-{{key}}-explorer-pg" {% endif %} +{% if value['USE_PLACEMENT_GROUP']|default('true') == "true" %} placement_group = "${var.prefix}-{{key}}-explorer-pg" {% endif %} launch_configuration = "${aws_launch_configuration.explorer.name}" vpc_zone_identifier = ["${aws_subnet.default.id}"] availability_zones = ["${data.aws_availability_zones.available.names}"] diff --git a/roles/main_infra/templates/routing.tf.j2 b/roles/main_infra/templates/routing.tf.j2 index 0051e99..7593d1c 100644 --- a/roles/main_infra/templates/routing.tf.j2 +++ b/roles/main_infra/templates/routing.tf.j2 @@ -63,7 +63,7 @@ resource "aws_alb_listener" "alb_listener{{loop.index-1}}" { load_balancer_arn = "${aws_lb.explorer.*.arn[{{loop.index-1}}]}" port = "${lookup(var.use_ssl,element(var.chains,{{loop.index-1}})) ? "443" : "80" }" protocol = "${lookup(var.use_ssl,element(var.chains,{{loop.index-1}})) ? "HTTPS" : "HTTP" }" -{% if value['ECTO_USE_SSL'] == "true" %} +{% if value['ECTO_USE_SSL']|default('false') == "true" %} ssl_policy = "${lookup(var.alb_ssl_policy,element(var.chains,{{loop.index-1}}))}" certificate_arn = "${lookup(var.alb_certificate_arn,element(var.chains,{{loop.index-1}}))}" {% endif %} diff --git a/roles/main_infra/templates/terraform.tfvars.j2 b/roles/main_infra/templates/terraform.tfvars.j2 index f95c72d..c18a569 100644 --- a/roles/main_infra/templates/terraform.tfvars.j2 +++ b/roles/main_infra/templates/terraform.tfvars.j2 @@ -10,19 +10,19 @@ root_block_size = "{{ root_block_size }}" pool_size = { {% for key, value in chain_custom_environment.iteritems() %} -{{ key }}="{{ value['POOL_SIZE'] }}"{% if not loop.last %},{% endif %} +{{ key }}="{{ value['POOL_SIZE']|default('30') }}"{% if not loop.last %},{% endif %} {% endfor %} } secret_key_base = { {% for key, value in chain_custom_environment.iteritems() %} -{{ key }}="{{ value['SECRET_KEY_BASE'] }}"{% if not loop.last %},{% endif %} +{{ key }}="{{ value['SECRET_KEY_BASE']|default('TPGMvGK0iIwlXBQuQDA5KRqk77VETbEBlG4gAWeb93TvBsYAjvoAvdODMd6ZeguPwf2YTRY3n7uvxXzQP4WayQ==') }}"{% if not loop.last %},{% endif %} {% endfor %} } use_ssl = { {% for key, value in chain_custom_environment.iteritems() %} -{{ key }}="{{ value['ECTO_USE_SSL'] }}"{% if not loop.last %},{% endif %} +{{ key }}="{{ value['ECTO_USE_SSL']|default('false') }}"{% if not loop.last %},{% endif %} {% endfor %} } diff --git a/roles/main_software/tasks/main.yml b/roles/main_software/tasks/main.yml index 133d257..70b3f3b 100644 --- a/roles/main_software/tasks/main.yml +++ b/roles/main_software/tasks/main.yml @@ -175,7 +175,7 @@ - name: Upload output debug: - msg: "If deployment will fail, you can try to deploy blockscout manually using the following commands: {{ push_output.stdout_lines }}" + msg: "If deployment will fail, you can try to deploy blockscout manually using the following commands: {{ 'AWS_ACCESS_KEY=XXXXXXXXXXXXXX AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXX AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} {{ push_output.stdout_lines }} {{ '--profile='~aws_profile if aws_profile is defined else '' }}" - name: User prompt pause: From a4242b84c30f25ec288f312942d20b4bc5556fad Mon Sep 17 00:00:00 2001 From: "a@a.ru" Date: Wed, 24 Apr 2019 17:44:39 +0300 Subject: [PATCH 4/6] Add coinmarketcap pages variable --- group_vars/all.yml.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/group_vars/all.yml.example b/group_vars/all.yml.example index ae7323d..7d57b75 100644 --- a/group_vars/all.yml.example +++ b/group_vars/all.yml.example @@ -37,6 +37,7 @@ chain_custom_environment: #CHECK_ORIGIN: "false" # Used to check the origin of requests when the origin header is present PORT: 4000 # Port the application runs on COIN: "POA" # Coin name at the Coinmarketcap, used to display current exchange rate + #COINMARKETCAP_PAGES: 10 # Sets the number of pages at Coinmarketcap to search coin at. Defaults to 10 #METADATA_CONTRACT: # Address of metadata smart contract. Used by POA Network to obtain Validators information to display in the UI #VALIDATORS_CONTRACT: #Address of the EMission Fund smart contract #SUPPLY_MODULE: "false" # Used by the xDai Chain to calculate the total supply of the chain @@ -65,4 +66,4 @@ chain_custom_environment: USE_PLACEMENT_GROUP: "false" # If true, BlockScout instance will be created in the placement group EXQ_BLOCKS_CONCURRENCY: "1" EXQ_INTERNAL_TRANSACTIONS_CONCURRENCY: "1" - EXQ_TRANSACTIONS_CONCURRENCY: "1" + EXQ_TRANSACTIONS_CONCURRENCY: "1" From 3163217f0f6458d6577302ba4c391adf0a342846 Mon Sep 17 00:00:00 2001 From: "a@a.ru" Date: Tue, 30 Apr 2019 13:19:13 +0300 Subject: [PATCH 5/6] Another fix for favicon, add tags to ASG, remove obsolete vars --- group_vars/all.yml.example | 3 --- roles/main_infra/templates/hosts.tf.j2 | 6 ++++++ roles/main_software/tasks/main.yml | 9 ++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/group_vars/all.yml.example b/group_vars/all.yml.example index 7d57b75..98d52fd 100644 --- a/group_vars/all.yml.example +++ b/group_vars/all.yml.example @@ -64,6 +64,3 @@ chain_custom_environment: ADDRESS_WITH_BALANCES_UPDATE_INTERVAL: 1800 #Interval in seconds to restart the task, which calculates addresses with balances LINK_TO_OTHER_EXPLORERS: "false" # If true, links to other explorers are added in the footer USE_PLACEMENT_GROUP: "false" # If true, BlockScout instance will be created in the placement group - EXQ_BLOCKS_CONCURRENCY: "1" - EXQ_INTERNAL_TRANSACTIONS_CONCURRENCY: "1" - EXQ_TRANSACTIONS_CONCURRENCY: "1" diff --git a/roles/main_infra/templates/hosts.tf.j2 b/roles/main_infra/templates/hosts.tf.j2 index 666ef3f..4ed7daa 100644 --- a/roles/main_infra/templates/hosts.tf.j2 +++ b/roles/main_infra/templates/hosts.tf.j2 @@ -94,6 +94,12 @@ resource "aws_autoscaling_group" "explorer-{{key}}" { value = "{{ key }}" propagate_at_launch = true } + + tag { + key = "Name" + value = "{{ key }} Application" + propagate_at_launch = true + } } # TODO: These autoscaling policies are not currently wired up to any triggers diff --git a/roles/main_software/tasks/main.yml b/roles/main_software/tasks/main.yml index 70b3f3b..94bbae5 100644 --- a/roles/main_software/tasks/main.yml +++ b/roles/main_software/tasks/main.yml @@ -163,11 +163,10 @@ when: user_answer.user_input|lower != "false" and user_answer.user_input|lower != "no" - name: Fix bug with favicon - copy: - src: "{{ item }}" - dest: "blockscout-{{ chain }}/apps/block_scout_web/priv/static/images/" - with_fileglob: - - "blockscout-{{ chain }}/apps/block_scout_web/priv/static/favicon*" + replace: + regexp: 'favicon-[a-z0-9]+?\.ico' + replace: "favicon.ico" + path: "blockscout-{{ chain }}/apps/block_scout_web/priv/static/cache_manifest.json" - name: Upload Blockscout to S3 command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} aws deploy push --application-name={{ prefix }}-explorer --s3-location s3://{{ prefix }}-explorer-codedeploy-releases/blockscout-{{ chain }}.zip --source=blockscout-{{ chain }} {{ '--profile='~aws_profile if aws_profile is defined else '' }}" From 19bf827ce2431a643751744ef9a23b8c88b10245 Mon Sep 17 00:00:00 2001 From: Arsenii Petrovich Date: Tue, 30 Apr 2019 13:31:00 +0300 Subject: [PATCH 6/6] Update all.yml.example --- group_vars/all.yml.example | 39 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/group_vars/all.yml.example b/group_vars/all.yml.example index 98d52fd..6d2314f 100644 --- a/group_vars/all.yml.example +++ b/group_vars/all.yml.example @@ -21,7 +21,7 @@ bucket: "poa-terraform-state" ## All resources will be prefixed with this one prefix: "poa" -## This variable represents network RPC endpoint: +## This dictionary represents a set of environment variables required for each chain. Variables that commented out are optional. chain_custom_environment: core: NETWORK: "(POA)" # Name of the organization/community that hosts the chain @@ -34,33 +34,34 @@ chain_custom_environment: ETHEREUM_JSONRPC_WS_URL: "ws://localhost:8546" # Network RPC endpoint in websocket mode NETWORK_PATH: "/poa/core" # relative URL path, for example: blockscout.com/$NETWORK_PATH SECRET_KEY_BASE: "TPGMvGK0iIwlXBQuQDA5KRqk77VETbEBlG4gAWeb93TvBsYAjvoAvdODMd6ZeguPwf2YTRY3n7uvxXzQP4WayQ==" # Secret key for production assets protection. Use `mix phx.gen.secret` or `openssl rand -base64 64 | tr -d '\n'` to generate - #CHECK_ORIGIN: "false" # Used to check the origin of requests when the origin header is present PORT: 4000 # Port the application runs on COIN: "POA" # Coin name at the Coinmarketcap, used to display current exchange rate + POOL_SIZE: 20 # Defines the number of database connections allowed + ECTO_USE_SSL: "false" # Specifies whether or not to use SSL on Ecto queries + ALB_SSL_POLICY: "ELBSecurityPolicy-2016-08" #SSL policy for Load Balancer. Required if ECTO_USE_SSL is set to true + ALB_CERTIFICATE_ARN: "arn:aws:acm:us-east-1:290379793816:certificate/6d1bab74-fb46-4244-aab2-832bf519ab24" #ARN of the certificate to attach to the LB. Required if ECTO_USE_SSL is set to + true + HEART_BEAT_TIMEOUT: 30 # Heartbeat is an Erlang monitoring service that will restart BlockScout if it becomes unresponsive. This variables configures the timeout before Blockscout will be restarted. + HEART_COMMAND: "sudo systemctl restart explorer.service" # This variable represents a command that is used to restart the service + BLOCKSCOUT_VERSION: "v1.3.11-beta" # Added to the footer to signify the current BlockScout version + RELEASE_LINK: "https://github.com/poanetwork/blockscout/releases/tag/v1.3.9-beta" # The link to Blockscout release notes in the footer. + ELIXIR_VERSION: "v1.8.1" # Elixir version to install on the node before Blockscout deploy + BLOCK_TRANSFORMER: "base" # Transformer for blocks: base or clique. + GRAPHIQL_TRANSACTION: "0xbc426b4792c48d8ca31ec9786e403866e14e7f3e4d39c7f2852e518fae529ab4" # Random tx hash on the network, used as default for graphiql tx. + TXS_COUNT_CACHE_PERIOD: 7200 # Interval in seconds to restart the task, which calculates the total txs count. + ADDRESS_WITH_BALANCES_UPDATE_INTERVAL: 1800 #Interval in seconds to restart the task, which calculates addresses with balances + LINK_TO_OTHER_EXPLORERS: "false" # If true, links to other explorers are added in the footer + USE_PLACEMENT_GROUP: "false" # If true, BlockScout instance will be created in the placement group + #The following variables are optional + #FIRST_BLOCK: 0 # The block number, where indexing begins from. #COINMARKETCAP_PAGES: 10 # Sets the number of pages at Coinmarketcap to search coin at. Defaults to 10 #METADATA_CONTRACT: # Address of metadata smart contract. Used by POA Network to obtain Validators information to display in the UI #VALIDATORS_CONTRACT: #Address of the EMission Fund smart contract #SUPPLY_MODULE: "false" # Used by the xDai Chain to calculate the total supply of the chain #SOURCE_MODULE: "false" # Used to calculate the total supply #DATABASE_URL: # Database URL. Usually generated automatically, but this variable can be used to modify the URL of the databases during the updates. - POOL_SIZE: 20 # Defines the number of database connections allowed - ECTO_USE_SSL: "false" # Specifies whether or not to use SSL on Ecto queries - ALB_SSL_POLICY: "ELBSecurityPolicy-2016-08" #SSL policy for Load Balancer. Required if ECTO_USE_SSL is set to true - ALB_CERTIFICATE_ARN: "arn:aws:acm:us-east-1:290379793816:certificate/6d1bab74-fb46-4244-aab2-832bf519ab24" #ARN of the certificate to attach to the LB. Required if ECTO_USE_SSL is set to - true + #CHECK_ORIGIN: "false" # Used to check the origin of requests when the origin header is present #DATADOG_HOST: # Host configuration variable for Datadog integration #DATADOG_PORT: # Port configuration variable for Datadog integration #SPANDEX_BATCH_SIZE: # Spandex and Datadog configuration setting. #SPANDEX_SYNC_THRESHOLD: # Spandex and Datadog configuration setting. - HEART_BEAT_TIMEOUT: 30 # Heartbeat is an Erlang monitoring service that will restart BlockScout if it becomes unresponsive. This variables configures the timeout before Blockscout will be restarted. - HEART_COMMAND: "sudo systemctl restart explorer.service" # This variable represents a command that is used to restart the service - BLOCKSCOUT_VERSION: "v1.3.9" # Added to the footer to signify the current BlockScout version - RELEASE_LINK: "https://github.com/poanetwork/blockscout/releases/tag/v1.3.9-beta" # The link to Blockscout release notes in the footer. - ELIXIR_VERSION: "v1.8.1" # Elixir version to install on the node before Blockscout deploy - BLOCK_TRANSFORMER: "base" # Transformer for blocks: base or clique. - GRAPHIQL_TRANSACTION: "0xbc426b4792c48d8ca31ec9786e403866e14e7f3e4d39c7f2852e518fae529ab4" # Random tx hash on the network, used as default for graphiql tx. - FIRST_BLOCK: 0 # The block number, where indexing begins from. - TXS_COUNT_CACHE_PERIOD: 7200 # Interval in seconds to restart the task, which calculates the total txs count. - ADDRESS_WITH_BALANCES_UPDATE_INTERVAL: 1800 #Interval in seconds to restart the task, which calculates addresses with balances - LINK_TO_OTHER_EXPLORERS: "false" # If true, links to other explorers are added in the footer - USE_PLACEMENT_GROUP: "false" # If true, BlockScout instance will be created in the placement group