Several user improvements

.gitignore

@@ -7,6 +7,7 @@ roles/main_infra/files/remote-backend-selector.tf
 roles/main_infra/files/terraform.tfvars
 roles/main_infra/files/hosts.tf
 roles/main_infra/files/routing.tf
+roles/main_infra/files/provider.tf
 *.backup
 
 # Sensitive information

attach_existing_rds.yml

@@ -9,7 +9,3 @@
     - name: "confirmation"
       prompt: "Are you sure you want to attach the existing RDS? If backend variable is set to True, this action includes creating the S3 and DynamoDB table for storing Terraform state files."
       default: False
-  environment:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"

deploy_infra.yml

@@ -5,7 +5,3 @@
     - { role: s3, when: "backend|bool == true" }
     - { role: dynamodb, when: "backend|bool == true" }
     - { role: main_infra }
-  environment:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"

deploy_software.yml

@@ -2,10 +2,6 @@
   hosts: localhost
   roles:
     - { role: s3, when: "backend|bool == true" }
-  environment:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"
 
 - name: Deploy BlockScout
   hosts: localhost

destroy.yml

@@ -6,7 +6,3 @@
     - name: "confirmation"
       prompt: "Are you sure you want to destroy all the infra?"
       default: False
-  environment:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"

group_vars/all.yml.example

@@ -3,9 +3,11 @@ ansible_python_interpreter: "/usr/bin/python3"
 
 # Common variables
 
-## Credentials to connect to AWS
+## Credentials to connect to AWS. Either keypair or CLI profile name should be specified. If nothing is specified, the default AWS keypair is used. Region must be specified in all the cases.
-aws_access_key: ""
+#aws_access_key: ""
-aws_secret_key: ""
+#aws_secret_key: ""
+#aws_profile: ""
+aws_region: "us-east-1"
 
 ## If set to true backend will be uploaded and stored at S3 bucket, so you can easily manage your deployment from any machine. It is highly recommended to do not change this variable
 backend: true
@@ -61,9 +63,6 @@ chain_custom_environment:
     ADDRESS_WITH_BALANCES_UPDATE_INTERVAL: 1800 #Interval in seconds to restart the task, which calculates addresses with balances
     LINK_TO_OTHER_EXPLORERS: "false" # If true, links to other explorers are added in the footer
     USE_PLACEMENT_GROUP: "false" # If true, BlockScout instance will be created in the placement group
-    EXQ_BLOCKS_CONCURRENCY="1"
+    EXQ_BLOCKS_CONCURRENCY: "1"
-    EXQ_INTERNAL_TRANSACTIONS_CONCURRENCY="1"
+    EXQ_INTERNAL_TRANSACTIONS_CONCURRENCY: "1"
-    EXQ_TRANSACTIONS_CONCURRENCY="1"
+    EXQ_TRANSACTIONS_CONCURRENCY: "1"
-
-## Region. It is recommended to deploy to us-east-1 as some of the other regions fails due to varied reasons
-region: "us-east-1"

group_vars/infrastructure.yml.example

@@ -24,9 +24,6 @@ dns_zone_name: "poa.internal"
 ## Size of the EC2 instance EBS root volume
 root_block_size: 120
 
-## The following variable represents the elixir version that will be used to run BlockScout instance. Will be used to download the Elixir at the following link: https://github.com/elixir-lang/elixir/releases/download/{{ elixir_version }}/Precompiled.zip
-elixir_version: "v1.7.4"
-
 # DB related variables
 
 ## This value represents the name of the DB that will be created/attached. Must be unique. Will be prefixed with `prefix` variable.

roles/attach_existing_rds/tasks/main.yml

@@ -16,6 +16,8 @@
   template:
     src: roles/main_infra/templates/terraform.tfvars.j2
     dest: roles/main_infra/files/terraform.tfvars
+  vars:
+    db_iops: "{{ chain_db_iops | default({}) }}"
 
 - name: Generating backend file
   template:
@@ -23,6 +25,12 @@
     dest: roles/main_infra/files/backend.tfvars
   when: backend|bool == true
 
+- name: Generate Terraform files
+  template:
+    src: "{{ item.key }}"
+    dest: "{{ item.value }}"
+  with_dict: {roles/main_infra/templates/hosts.tf.j2: roles/main_infra/files/hosts.tf,roles/main_infra/templates/routing.tf.j2: roles/main_infra/files/routing.tf,roles/main_infra/templates/provider.tf.j2: roles/main_infra/files/provider.tf}
+
 #Workaround since terraform module return unexpected error.
 - name: Initialize Terraform
   shell: "echo yes | {{ terraform_location }} init{{ ' -backend-config=backend.tfvars' if backend|bool == true else '' }}"

roles/destroy/tasks/main.yml

@@ -16,6 +16,8 @@
   template:
     src: roles/main_infra/templates/terraform.tfvars.j2
     dest: roles/main_infra/files/terraform.tfvars
+  vars:
+    db_iops: "{{ chain_db_iops | default({}) }}"
 
 - name: Generating backend file
   template:
@@ -23,6 +25,12 @@
     dest: roles/main_infra/files/backend.tfvars
   when: backend|bool == true
 
+- name: Generate Terraform files
+  template:
+    src: "{{ item.key }}"
+    dest: "{{ item.value }}"
+  with_dict: {roles/main_infra/templates/hosts.tf.j2: roles/main_infra/files/hosts.tf,roles/main_infra/templates/routing.tf.j2: roles/main_infra/files/routing.tf,roles/main_infra/templates/provider.tf.j2: roles/main_infra/files/provider.tf}
+
 # This is due to the TF0.11 bug which do not allow to completely destroy resources if interpolation syntax is used in outputs.tf at edge cases
 - name: Check if outputs.tf exists
   stat: path=roles/main_infra/files/outputs.tf
@@ -40,6 +48,13 @@
     - "init {{ '-backend-config=backend.tfvars' if backend|bool == true else '' }}"
     - destroy
 
+- name: Delete vars from parameter store
+  include: parameter_store.yml
+  loop: "{{ chain_custom_environment.keys() }}"
+  loop_control:
+    loop_var: chain
+    index_var: index
+
 - name: Check if outputs.tf.backup exists
   stat: path=roles/main_infra/files/outputs.tf.backup
   register: outputs_backup_stat
@@ -58,9 +73,27 @@
     name: "{{ prefix }}-{{ bucket }}"
     state: absent
     force: yes
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
   when: user_answer.user_input|bool == True
 
 - dynamodb_table:
     name: "{{ prefix }}-{{ dynamodb_table }}"
     state: absent
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
   when: user_answer.user_input|bool == True

roles/destroy/tasks/parameter_store.yml

@@ -0,0 +1,29 @@
+- name: Fetch environment variables (via access key)
+  set_fact:
+    chain_env: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=region, shortnames=true, bypath=true, recursive=true  ) }}"
+  vars:
+    path: "/{{ prefix }}/{{ chain }}"
+  when: aws_access_key is defined
+
+- name: Fetch environment variables (via profile)
+  set_fact:
+    chain_env: "{{ lookup('aws_ssm', path, aws_profile=aws_profile, shortnames=true, bypath=true, recursive=true  ) }}"
+  vars:
+    path: "/{{ prefix }}/{{ chain }}"
+  when: aws_profile is defined
+
+- name: Remove chain variables
+  aws_ssm_parameter_store:
+    name: "/{{ prefix }}/{{ chain }}/{{ item.key }}"    
+    value: "{{ item.value }}"
+    state: absent
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"    
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
+  with_dict: "{{ chain_env }}"

roles/dynamodb/tasks/main.yml

@@ -8,3 +8,12 @@
     tags:
       origin: terraform
       prefix: "{{ prefix }}"
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"

roles/main_infra/files/provider.tf

@@ -1,5 +0,0 @@
-provider "aws" {
-  version = "~> 1.15"
-
-  region = "${var.region}"
-}

roles/main_infra/files/variables.tf

@@ -7,13 +7,15 @@ variable "db_subnet_cidr" {}
 variable "dns_zone_name" {}
 variable "instance_type" {}
 variable "root_block_size" {}
+
 variable "pool_size" {
   default = {} 
 }
-variable "elixir_version" {}
+
 variable "use_placement_group" {
   default = {}
 }
+
 variable "key_content" {
   default = ""
 }

roles/main_infra/tasks/main.yml

@@ -40,7 +40,7 @@
   template:
     src: "{{ item.key }}"
     dest: "{{ item.value }}"
-  with_dict: {hosts.tf.j2: roles/main_infra/files/hosts.tf,routing.tf.j2: roles/main_infra/files/routing.tf}
+  with_dict: {hosts.tf.j2: roles/main_infra/files/hosts.tf,routing.tf.j2: roles/main_infra/files/routing.tf,provider.tf.j2: roles/main_infra/files/provider.tf}
     
 #Workaround since terraform module return unexpected error.
 - name: Terraform plan construct

roles/main_infra/tasks/parameter_store.yml

@@ -9,8 +9,13 @@
   aws_ssm_parameter_store:
     name: "/{{ prefix }}/{{ chain }}/{{ item.key }}"
     value: "{{ item.value }}"
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
   with_dict: "{{ chain_ps_env }}"
-  environment:
+  vars:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+    access_key: "{{ aws_access_key|default(omit) }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
-    AWS_REGION: "{{ region }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"

roles/main_infra/templates/backend.tfvars.j2

@@ -1,4 +1,3 @@
-region         = "{{ ansible_env.AWS_REGION }}"
 bucket         = "{{ prefix }}-{{ bucket }}"
 dynamodb_table = "{{ prefix }}-{{ dynamodb_table }}"
 key            = "terraform.tfstate"

roles/main_infra/templates/hosts.tf.j2

@@ -40,10 +40,12 @@ resource "aws_launch_configuration" "explorer" {
 }
 
 {% for key, value in chain_custom_environment.iteritems() %}
+{% if value['USE_PLACEMENT_GROUP'] == "true" %}
 resource "aws_placement_group" "explorer-{{key}}" {
   name     = "${var.prefix}-{{key}}-explorer-pg"
   strategy = "cluster"
 }
+{% endif %}
 {% endfor %}
 
 {% for key, value in chain_custom_environment.iteritems() %}

roles/main_infra/templates/provider.tf.j2

@@ -0,0 +1,7 @@
+provider "aws" {
+  version  = "~> 1.15"
+{% if aws_access_key is undefined %}
+  profile  = "{{ aws_profile|default("default") }}"
+{% endif %}
+  region   = "{{ aws_region|default("us-east-1") }}"
+}

roles/main_infra/templates/remote-backend-selector.tf.j2

@@ -1,4 +1,11 @@
 terraform {
   backend "s3" {
+    {% if aws_access_key is undefined %}
+      profile    = "{{ aws_profile|default("default") }}"
+    {% else %}
+      access_key = "{{ aws_access_key }}"
+      secret_key = "{{ aws_secret_key }}"
+    {% endif %}
+      region     = "{{ aws_region }}"
   }
 }

roles/main_infra/templates/terraform.tfvars.j2

@@ -1,14 +1,12 @@
-region                = "{{ ansible_env.AWS_REGION }}"
+prefix             = "{{ prefix }}"
-prefix                = "{{ prefix }}"
+key_name           = "{{ ec2_ssh_key_name }}"
-key_name              = "{{ ec2_ssh_key_name }}"
+key_content        = "{{ ec2_ssh_key_content }}"
-key_content           = "{{ ec2_ssh_key_content }}"
+vpc_cidr           = "{{ vpc_cidr }}"
-vpc_cidr              = "{{ vpc_cidr }}"
+public_subnet_cidr = "{{ public_subnet_cidr  }}"
-public_subnet_cidr    = "{{ public_subnet_cidr  }}"
+db_subnet_cidr     = "{{ db_subnet_cidr }}"
-db_subnet_cidr        = "{{ db_subnet_cidr }}"
+dns_zone_name      = "{{ dns_zone_name }}"
-dns_zone_name         = "{{ dns_zone_name }}"
+instance_type      = "{{ instance_type }}"
-instance_type         = "{{ instance_type }}"
+root_block_size    = "{{ root_block_size }}"
-root_block_size       = "{{ root_block_size }}"
-elixir_version        = "{{ elixir_version }}"
 
 pool_size = {
 {% for key, value in chain_custom_environment.iteritems() %}
@@ -28,15 +26,16 @@ use_ssl = {
 {% endfor %}
 }
 
+
 alb_ssl_policy = {
 {% for key, value in chain_custom_environment.iteritems() %}
-{{ key }}="{{ value['ALB_SSL_POLICY'] }}"{% if not loop.last %},{% endif %}
+{{ key }}="{{ value['ALB_SSL_POLICY']|default('') }}"{% if not loop.last %},{% endif %}
 {% endfor %}
 }
 
 alb_certificate_arn = {
 {% for key, value in chain_custom_environment.iteritems() %}
-{{ key }}="{{ value['ALB_CERTIFICATE_ARN'] }}"{% if not loop.last %},{% endif %}
+{{ key }}="{{ value['ALB_CERTIFICATE_ARN']|default('') }}"{% if not loop.last %},{% endif %}
 {% endfor %}
 }
 

roles/main_software/tasks/main.yml

@@ -30,6 +30,13 @@
   template:
     src: dev.secret.exs.j2 
     dest: "blockscout-{{ chain }}/apps/explorer/config/dev.secret.exs" 
+  when: ps_db is defined 
+
+- name: Copy default explorer config files
+  copy:
+    src: "blockscout-{{ chain }}/apps/explorer/config/dev.secret.exs.example"
+    dest: "blockscout-{{ chain }}/apps/explorer/config/dev.secret.exs"
+  when: ps_db is undefined or ps_db == ""
 
 - name: Remove static assets from previous deployment, if any
   file:
@@ -69,24 +76,34 @@
   args:
     chdir: "blockscout-{{ chain }}/apps/block_scout_web"
 
-- name: Fetch environment variables
+- name: Fetch environment variables (via access key)
   set_fact: 
     chain_env: "{{ lookup('aws_ssm', path, aws_access_key=aws_access_key, aws_secret_key=aws_secret_key, region=region, shortnames=true, bypath=true, recursive=true  ) }}"
   vars:
     path: "/{{ prefix }}/{{ chain }}"
+  when: aws_access_key is defined
+
+- name: Fetch environment variables (via profile)
+  set_fact:
+    chain_env: "{{ lookup('aws_ssm', path, aws_profile=aws_profile, shortnames=true, bypath=true, recursive=true  ) }}"
+  vars:
+    path: "/{{ prefix }}/{{ chain }}"
+  when: aws_profile is defined
 
 - name: Make config variables lowercase
   set_fact:
     chain_lower_env: "{{ chain_lower_env | combine ({item.key|lower : item.value})  }}"
   with_dict: "{{ chain_custom_environment_chain }}"
+  when: chain_custom_environment_chain|length > 0
   vars:
     chain_lower_env: {}
-    chain_custom_environment_chain: "{{ chain_cec[chain] | default({}) }}"
+    chain_custom_environment_chain: "{{ chain_cec[chain] | default({}) if chain_cec[chain]>0 else {} }}"
     chain_cec: "{{ chain_custom_environment | default ({}) }}"
 
 - name: Override env variables
   set_fact:
     chain_env: "{{ chain_env | combine(chain_lower_env) }}"
+  when: chain_lower_env is defined
 
 - name: Uppercase chain 
   set_fact:
@@ -115,11 +132,13 @@
         msg: "Execution aborted."
   always:
     - name: kill server
-      command: "pkill -e {{ item }}"
+      command: "pkill -f {{ item }}"
       with_items:
         - beam.smp
         - node
+        - erlang
       failed_when: false  
+      when: 
  
 - name: Build static assets
   command: mix phx.digest
@@ -151,12 +170,8 @@
     - "blockscout-{{ chain }}/apps/block_scout_web/priv/static/favicon*"
 
 - name: Upload Blockscout to S3
-  command: "aws deploy push --application-name={{ prefix }}-explorer --s3-location s3://{{ prefix }}-explorer-codedeploy-releases/blockscout-{{ chain }}.zip --source=blockscout-{{ chain }}"
+  command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} aws deploy push --application-name={{ prefix }}-explorer --s3-location s3://{{ prefix }}-explorer-codedeploy-releases/blockscout-{{ chain }}.zip --source=blockscout-{{ chain }} {{ '--profile='~aws_profile if aws_profile is defined else '' }}"
   register: push_output
-  environment:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"
 
 - name: Upload output
   debug:
@@ -171,11 +186,17 @@
   aws_ssm_parameter_store:
     name: "/{{ prefix }}/{{ chain }}/{{ item.key }}" 
     value: "{{ item.value }}"
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
   with_dict: "{{ chain_env }}"
-  environment:
+  
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"
   when: user_answer.user_input|lower != "false" and user_answer.user_input|lower != "no"
 
 - name: User prompt
@@ -184,9 +205,5 @@
   register: user_answer
 
 - name: Deploy Blockscout
-  command: "{{ push_output.stdout_lines[1] }} --deployment-group-name {{ prefix }}-explorer-dg{{ index }} --deployment-config-name CodeDeployDefault.OneAtATime --description '{{ chain_upper_env['BLOCKSCOUT_VERSION']  }}'" 
+  command: "{{ 'AWS_ACCESS_KEY='~aws_access_key~' AWS_SECRET_ACCESS_KEY='~aws_secret_key~' AWS_DEFAULT_REGION='~aws_region if aws_profile is undefined else '' }} {{ push_output.stdout_lines[1] }} --deployment-group-name {{ prefix }}-explorer-dg{{ index }} --deployment-config-name CodeDeployDefault.OneAtATime --description '{{ chain_upper_env['BLOCKSCOUT_VERSION']  }}' {{ '--profile='~aws_profile if aws_profile is defined else '' }}" 
   when: user_answer.user_input|lower != "false" and user_answer.user_input|lower != "no"
-  environment:
-    AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
-    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
-    AWS_REGION: "{{ region }}"

roles/s3/tasks/main.yml

@@ -3,6 +3,15 @@
     bucket: "{{ prefix }}-{{ bucket }}" 
     mode: create
     permission: private
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
 
 - name: Apply tags and versioning to create S3 bucket
   s3_bucket:
@@ -11,6 +20,15 @@
     tags:
       origin: terraform
       prefix: "{{ prefix }}"
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
 
 - name: Add lifecycle management policy to created S3 bucket
   s3_lifecycle:
@@ -19,6 +37,15 @@
     noncurrent_version_expiration_days: 90
     status: enabled
     state: present
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
 
 - name: Check if config file exists 
   stat:
@@ -50,6 +77,15 @@
     object: all.yml
     src: "{{ playbook_dir }}/group_vars/all.yml.temp"
     mode: put
+    profile: "{{ profile }}"
+    aws_access_key: "{{ access_key }}"
+    aws_secret_key: "{{ secret_key }}"
+    region: "{{ region }}"
+  vars:
+    access_key: "{{ aws_access_key|default(omit) }}"
+    secret_key: "{{ aws_secret_key|default(omit) }}"
+    profile: "{{ aws_profile|default(omit) }}"
+    region: "{{ aws_region|default(omit) }}"
   when: upload_config_to_s3|bool == True and stat_result.stat.exists == True
 
 - name: Remove temp file