Merge remote-tracking branch 'upstream/master' into newfixcompat

Vagrantfile

@@ -3,7 +3,7 @@ ENV["LC_ALL"] = "en_US.UTF-8"
 
 Vagrant.configure("2") do |config|
 
-  servers = [ "validator", "explorer", "moc", "bootnode", "netstat" ]
+  servers = [ "validator", "explorer", "moc", "bootnode", "netstat", "blockscout" ]
 
   platform_os = ENV["poa_platform"]
   if platform_os == "ubuntu"
@@ -14,9 +14,21 @@ Vagrant.configure("2") do |config|
     platform = "ubuntu/xenial64"
   end
 
+  plugin_installed = false
+  unless Vagrant.has_plugin? ("vagrant-disksize")
+    system "vagrant plugin install vagrant-disksize"
+    plugin_installed = true
+  end
+
+  # Restart Vagrant when new plugin is installed
+  if plugin_installed === true
+    exec "vagrant #{ARGV.join' '}"
+  end
+
   servers.each do |machine|
     config.vm.define machine do |node|
       node.vm.box = platform
+      node.disksize.size = '100GB'
       node.vm.hostname = machine
 
       node.vm.provision :ansible do |ansible|
@@ -28,8 +40,9 @@ Vagrant.configure("2") do |config|
           "netstat" => ["netstat"],
           "moc" => ["moc"],
           "bootnode" => ["bootnode"],
+          "blockscout" => ["blockscout"],
         }
-        ansible.groups[platform_os] = [ "validator", "explorer", "netstat", "moc", "bootnode" ]        
+        ansible.groups[platform_os] = [ "validator", "explorer", "netstat", "moc", "bootnode", "blockscout" ]
       end
 
       node.vm.provision :shell do |shell|

aws/blockscout.yml

@@ -0,0 +1,74 @@
+---
+- name: Create blockscout security group
+  hosts: localhost
+  gather_facts: False
+  tasks:
+  - name: Create Security group
+    ec2_group:
+      ec2_access_key: "{{ access_key }}"
+      ec2_secret_key: "{{ secret_key }}"
+      name: "{{ blockscout_security_group }}"
+      description: "Default security group"
+      region: "{{ region }}"
+      purge_rules_egress: false
+      purge_rules: false
+      vpc_id: "{{ vpc_id | default('') }}"
+      rules:
+      - proto: tcp
+        from_port: 22
+        to_port: 22
+        cidr_ip: 0.0.0.0/0
+  tags: blockscout
+
+- name: Create blockscout
+  hosts: localhost
+  gather_facts: False
+  vars:
+    volumes:
+    - device_name: /dev/sda1
+      volume_size: 128
+      delete_on_termination: true
+  tasks:
+  - name: Launch instance
+    ec2:
+      id: "{{ blockscout_instance_name }}"
+      ec2_access_key: "{{ access_key }}"
+      ec2_secret_key: "{{ secret_key }}"
+      key_name: "{{ awskeypair_name }}"
+      instance_tags:
+        Name: "{{ blockscout_instance_name }}"
+      group: "{{ blockscout_security_group }}"
+      instance_type: "{{ blockscout_instance_type }}"
+      image: "{{ image }}"
+      count: "{{ blockscout_count_instances }}"
+      wait: yes
+      region: "{{ region }}"
+      vpc_subnet_id: "{{ vpc_subnet_id | default('') }}"
+      volumes: "{{ volumes }}"
+      assign_public_ip: yes
+    register:   ec2
+  - name: Add new instance to host group
+    add_host: hostname={{ item.public_ip }} groupname=launched
+    with_items: "{{ ec2.instances }}"
+  - name: Wait for SSH to come up
+    wait_for: host={{ item.public_ip }} port=22 delay=90 timeout=320 state=started
+    with_items: "{{ ec2.instances }}"
+    when: ec2.changed
+  tags: blockscout
+
+- name: Installing python
+  hosts: launched
+  gather_facts: False
+  user: ubuntu
+  become: True
+  vars:
+    ansible_ssh_port: 22
+  tasks:
+  - name: Install python
+    raw: test -e /usr/bin/python || (sudo apt -y update && sudo apt install -y python-minimal)
+
+- name: EC2 access setup
+  hosts: launched
+  become: False
+  roles:
+  - blockscout-access

aws/explorer.yml

@@ -55,7 +55,7 @@
       wait_for: host={{ item.public_ip }} port=22 delay=90 timeout=320 state=started
       with_items: "{{ ec2.instances }}"
       when: ec2.changed
-tags: explorer
+  tags: explorer
 
 - name: Installing python
   hosts: launched

aws/group_vars/all.yml.example

@@ -106,3 +106,16 @@ validator_count_instances: "1"
 
 validator_security_group: "{{ MAIN_REPO_FETCH }}-{{ GENESIS_BRANCH }}-validator-security"
 associate_validator_elastic_ip: false
+
+###Blockscout
+
+blockscout_instance_type: "t2.large"
+blockscout_instance_name: "blockscout"
+blockscout_count_instances: "1"
+
+blockscout_security_group: "{{ MAIN_REPO_FETCH }}-{{ GENESIS_BRANCH }}-blockscout-security"
+
+allow_blockscout_ssh: true
+allow_blockscout_p2p: true
+allow_blockscout_http: false
+

aws/roles/blockscout-access/tasks/ec2.yml

@@ -0,0 +1,92 @@
+---
+- name: Create blockscout security group
+  delegate_to: localhost
+  ec2_group:
+    ec2_access_key: "{{ access_key }}"
+    ec2_secret_key: "{{ secret_key }}"
+    name: "{{ blockscout_security_group }}"
+    description: "Default security group"
+    region: "{{ region }}"
+    purge_rules: true
+    vpc_id: "{{ vpc_id | default('') }}"
+
+- name: Add ssh access
+  delegate_to: localhost
+  ec2_group:
+    ec2_access_key: "{{ access_key }}"
+    ec2_secret_key: "{{ secret_key }}"
+    name: "{{ blockscout_security_group }}"
+    description: "Default security group"
+    region: "{{ region }}"
+    purge_rules_egress: false
+    purge_rules: false
+    vpc_id: "{{ vpc_id | default('') }}"
+    rules:
+    - proto: tcp
+      from_port: "{{ item }}"
+      to_port: "{{ item }}"
+      cidr_ip: 0.0.0.0/0
+  with_items:
+  - "22"
+  when: allow_blockscout_ssh == true
+
+- name: Add https access
+  delegate_to: localhost
+  ec2_group:
+    ec2_access_key: "{{ access_key }}"
+    ec2_secret_key: "{{ secret_key }}"
+    name: "{{ blockscout_security_group }}"
+    description: "Default security group"
+    region: "{{ region }}"
+    purge_rules_egress: false
+    purge_rules: false
+    vpc_id: "{{ vpc_id | default('') }}"
+    rules:
+    - proto: tcp
+      from_port: 443
+      to_port: 443
+      cidr_ip: 0.0.0.0/0
+
+- name: Add p2p access
+  delegate_to: localhost
+  ec2_group:
+    ec2_access_key: "{{ access_key }}"
+    ec2_secret_key: "{{ secret_key }}"
+    name: "{{ blockscout_security_group }}"
+    description: "Default security group"
+    region: "{{ region }}"
+    purge_rules_egress: false
+    purge_rules: false
+    vpc_id: "{{ vpc_id | default('') }}"
+    rules:
+    - proto: tcp
+      from_port: "{{ item }}"
+      to_port: "{{ item }}"
+      cidr_ip: 0.0.0.0/0
+    - proto: udp
+      from_port: "{{ item }}"
+      to_port: "{{ item }}"
+      cidr_ip: 0.0.0.0/0
+  with_items:
+  - "30303"
+  when: allow_blockscout_p2p == true
+
+- name: Add http-4000 access
+  delegate_to: localhost
+  ec2_group:
+    ec2_access_key: "{{ access_key }}"
+    ec2_secret_key: "{{ secret_key }}"
+    name: "{{ blockscout_security_group }}"
+    description: "Default security group"
+    region: "{{ region }}"
+    purge_rules_egress: false
+    purge_rules: false
+    vpc_id: "{{ vpc_id | default('') }}"
+    rules:
+    - proto: tcp
+      from_port: "{{ item }}"
+      to_port: "{{ item }}"
+      cidr_ip: 0.0.0.0/0
+  with_items:
+  - "4000"
+  when: allow_blockscout_http == true

aws/roles/blockscout-access/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Setup EC2 firewall
+  import_tasks: ec2.yml
+  when: ansible_bios_version is search("amazon")

blockscout.yml

@@ -0,0 +1,4 @@
+- hosts: blockscout
+  become: True
+  roles:
+  - blockscout

group_vars/all.yml.example

@@ -126,7 +126,7 @@ validator_archive: "off"
 configure_extended_tcp_settings: false
 #Setup extended nginx headers. Used in bootnode, explorer and netstat roles.
 nginx_headers: "on"
-#Setup which proxy port should be used in nginx. Used in bootnode, explorer and netstat roles. 
+#Setup which proxy port should be used in nginx. Used in bootnode, explorer, blockscout and netstat roles.
 PROXY_PORT: "8545"
 
 
@@ -174,3 +174,19 @@ api_version: "9773b5b"
 
 #Node's descriptive name. Publicly visible in netstats
 #NODE_FULLNAME: "INSERT HERE"
+
+
+##Blockscout-related variables
+
+#Explicitly specified api version to use
+blockscout_version: "44e4f17d14"
+#Name of database user to create
+db_user: "INSERT HERE"
+#Password for the database user
+db_user_password: "INSERT HERE"
+#Specifies whether or not to keep port 22 opened to allow ssh access
+allow_blockscout_ssh: true
+#Specifies whether or not to keep port 30303 opened to allow peer discovery
+allow_blockscout_p2p: true
+#Specifies whether or not to keep port 4000 opened to allow http access
+allow_blockscout_http: false

group_vars/blockscout.yml.example

@@ -0,0 +1,16 @@
+username: "blockscout"
+users:
+- name: "{{ username }}"
+home: "/home/{{ username }}"
+
+install_debug_tools: "no"
+reboot_after_completion: false
+
+blockscout_version: "44e4f17d14"
+db_user: "blockscout"
+db_user_password: ""
+PROXY_PORT: "4000"
+
+allow_blockscout_ssh: true
+allow_blockscout_p2p: true
+allow_blockscout_http: false

hosts.example

@@ -21,12 +21,15 @@ node03.example.com
 
 [explorer]
 
+[blockscout]
+
 [poa:children]
 bootnode
 moc
 netstat
 validator
 explorer
+blockscout
 
 [centos]
 node03.example.com

roles/blockscout/defaults/main.yml

@@ -0,0 +1,24 @@
+---
+
+user: ubuntu
+
+allow_blockscout_ssh: true
+allow_blockscout_p2p: true
+allow_blockscout_http: false
+
+username: "blockscout"
+users:
+- name: "{{ username }}"
+home: "/home/{{ username }}"
+
+MAIN_REPO_FETCH: "poanetwork"
+MOC_ADDRESS: "0xdd0bb0e2a1594240fed0c2f2c17c1e9ab4f87126"
+
+reboot_after_completion: true
+install_debug_tools: "no"
+install_firewall: false
+
+#Use branch with styles for specified network
+blockscout_version: "{{ 'f1aafe7406' if GENESIS_BRANCH == 'sokol' else 'a8c4c72c54' }}"
+db_user: "blockscout"
+PROXY_PORT: "4000"

roles/blockscout/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: restart poa-blockscout
+  service: name=poa-blockscout state=restarted
+
+- name: restart ufw
+  service: name=ufw state=restarted

roles/blockscout/meta/main.yml

@@ -0,0 +1,8 @@
+dependencies:
+- { role: usermanager }
+- { role: preconf }
+- { role: nodejs }
+- { role: poa-parity }
+- { role: nginx }
+- { role: elixir }
+- { role: debug-tools, when: "install_debug_tools == 'yes'" }

roles/blockscout/tasks/main.yml

@@ -0,0 +1,84 @@
+---
+
+- name: Create node.toml
+  template: src=node.toml.j2 dest={{ home }}/node.toml owner={{ username }} group={{ username }} mode=0644
+
+- name: Clone poanetwork/blockscout repo from GitHub
+  git:
+    repo: "https://github.com/{{ MAIN_REPO_FETCH }}/blockscout"
+    dest: "{{ home }}/blockscout"
+    version: "{{ blockscout_version }}"
+
+- name: Add explorer dev.secret
+  template:
+    src: dev.secret.j2
+    dest: "{{ home }}/blockscout/apps/explorer/config/dev.secret.exs"
+
+- name: Add block_scout_web dev.secret
+  copy:
+    src: '{{ home }}/blockscout/apps/block_scout_web/config/dev.secret.exs.example'
+    remote_src: yes
+    dest: '{{ home }}/blockscout/apps/block_scout_web/config/dev.secret.exs'
+    owner: '{{ username }}'
+    group: '{{ username }}'
+    mode: 0644
+
+- name: Update JSON RPC Variant
+  replace:
+    path: '{{ item }}'
+    regexp: 'ganache'
+    replace: "parity"
+  with_items:
+  - "{{ home }}/blockscout/apps/explorer/config/dev.exs"
+  - "{{ home }}/blockscout/apps/indexer/config/dev.exs"
+
+- name: Update default network name (for Core network)
+  lineinfile:
+    dest: '{{ home }}/blockscout/apps/block_scout_web/config/config.exs'
+    state: present
+    regexp: "^.*SUBNETWORK"
+    line: '  subnetwork: "POA Core",'
+  when: GENESIS_BRANCH == "core"
+
+- name: Install dependencies
+  shell: cd {{ home }}/blockscout && mix local.hex --force && mix do deps.get, local.rebar --force, deps.compile, compile
+
+- name: Create user
+  become: true
+  become_user: postgres
+  postgresql_user:
+    name: '{{ db_user }}'
+    password: '{{ db_user_password }}'
+    role_attr_flags: CREATEDB,SUPERUSER
+
+- name: Create db
+  shell: cd {{ home }}/blockscout && mix do ecto.create, ecto.migrate
+
+- name: Install node-sass
+  shell: cd {{ home }}/blockscout/apps/block_scout_web/assets && npm install node-sass --unsafe-perm
+
+- name: Install node dependensies in the apps/block_scout_web
+  npm:
+    path: "{{ home }}/blockscout/apps/block_scout_web/assets"
+    state: present
+
+- name: Install node dependensies in the apps/explorer
+  npm:
+    path: "{{ home }}/blockscout/apps/explorer"
+    state: present
+
+- name: Install poa-blockscout service
+  template: src=poa-blockscout.j2 dest=/etc/systemd/system/poa-blockscout.service owner={{ username }} group={{ username }} mode=0755
+  notify:
+  - restart poa-blockscout
+
+- name: Ensure poa-blockscout is running and enabled to start at boot
+  service: name=poa-blockscout state=started enabled=yes
+
+- name: Setup ufw firewall
+  import_tasks: ufw.yml
+  when: install_firewall
+
+- name: restart machine after setup
+  shell: shutdown -r 1
+  when: reboot_after_completion == true

roles/blockscout/tasks/ufw.yml

@@ -0,0 +1,51 @@
+---
+# to verify ufw configuration run:
+# sudo ufw status verbose
+
+- name: Install ufw
+  package: name=ufw state=present
+
+- name: Clean old rules
+  ufw: state=reset
+
+- name: Configure ufw defaults
+  ufw: direction={{ item.direction }} policy={{ item.policy }}
+  with_items:
+  - { direction: 'incoming', policy: 'deny' }
+  - { direction: 'outgoing', policy: 'allow' }
+  notify:
+  - restart ufw
+
+- name: Add ufw ssh access
+  ufw: rule='allow' port='22' proto='tcp'
+  notify:
+  - restart ufw
+  when: allow_blockscout_ssh == true
+
+- name: Add ufw https access
+  ufw: rule='allow' port='443' proto='tcp'
+  notify:
+  - restart ufw
+
+- name: Add ufw p2p access
+  ufw: rule={{ item.rule }} port={{ item.port }} proto={{ item.proto }}
+  with_items:
+  - { rule: 'allow', port: '30303', proto: 'tcp' }
+  - { rule: 'allow', port: '30303', proto: 'udp' }
+  notify:
+  - restart ufw
+  when: allow_explorer_p2p == true
+
+- name: Add ufw http-4000 access
+  ufw: rule='allow' port='4000' proto='tcp'
+  notify:
+  - restart ufw
+  when: allow_blockscout_http == true
+
+- name: Enable ufw logging
+  ufw: logging=off
+  notify:
+  - restart ufw
+
+- name: Enable ufw
+  ufw: state=enabled

roles/blockscout/templates/dev.secret.j2

@@ -0,0 +1,6 @@
+use Mix.Config
+
+config :explorer, Explorer.Repo,
+  username: "{{ db_user }}",
+  password: "{{ db_user_password }}"
+

roles/blockscout/templates/node.toml.j2

@@ -0,0 +1,35 @@
+[parity]
+chain = "spec.json"
+base_path = "parity_data"
+
+[rpc]
+apis = ["web3","eth","net", "parity",  "traces"]
+cors=["all"]
+
+[network]
+reserved_peers="{{ home }}/bootnodes.txt"
+nat="extip:{{ ansible_host }}"
+port = 30303
+max_peers = 100
+
+[websockets]
+disable = false
+port = 8546
+interface = "all"
+origins = ["all"]
+apis = ["web3","eth","net","parity", "pubsub", "traces"]
+hosts = ["all"]
+
+[ui]
+disable = true
+
+[snapshots]
+disable_periodic = true
+
+[footprint]
+tracing = "on"
+pruning = "archive"
+fat_db = "on"
+
+[misc]
+log_file = "{{ home }}/parity.log"

roles/blockscout/templates/poa-blockscout.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=poa blockscout service
+After=network.target
+[Service]
+User=root
+Group=root
+WorkingDirectory={{ home }}/blockscout
+Restart=on-failure
+RestartSec=5
+ExecStartPre=/usr/local/bin/mix local.hex --force
+ExecStart=/usr/local/bin/mix phx.server
+Type=simple
+[Install]
+WantedBy=multi-user.target

roles/elixir/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+
+ERLANG_SIGNING_KEY_URL: "https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc"
+ERLANG_SOURCE_DEB: "https://packages.erlang-solutions.com/ubuntu"
+POSTGRESQL_SIGNING_KEY_URL: "https://www.postgresql.org/media/keys/ACCC4CF8.asc"
+POSTGRESQL_SOURCE_DEB: "http://apt.postgresql.org/pub/repos/apt/"
+POSTGRESQL_DISTRIBUTION_RELEASE: "xenial-pgdg"
+ERLANG_RPM_URL: "http://packages.erlang-solutions.com/site/esl/esl-erlang/FLAVOUR_1_general/esl-erlang_21.0.5-1~centos~7_amd64.rpm"
+ELIXIR_REPO_URL: "git://github.com/elixir-lang/elixir.git"
+POSGRESQL_RPM_URL: "https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm"
+PG_HBA_PATH: "/var/lib/pgsql/10/data/pg_hba.conf"

roles/elixir/tasks/centos.yml

@@ -0,0 +1,68 @@
+---
+
+- name: Install epel-release, Erlang and other tools
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+  - automake
+  - libtool
+  - inotify-tools
+  - epel-release
+  - '{{ ERLANG_RPM_URL }}'
+
+- name: Download Elixir repo
+  git:
+    repo: '{{ ELIXIR_REPO_URL }}'
+    dest: /tmp/elixir
+    version: v1.7.1
+    accept_hostkey: yes
+    force: yes
+  register: elixir
+
+- name: Compile Elixir
+  shell:
+    make clean install chdir=/tmp/elixir
+  when: elixir.changed
+
+- name: Add symlinks
+  file:
+    src: '/usr/local/bin/{{ item.name }}'
+    dest: '/usr/bin/{{ item.name }}'
+    state: link
+  with_items:
+  - { name: 'mix' }
+  - { name: 'iex' }
+  - { name: 'elixir' }
+  - { name: 'elixirc' }
+  when: elixir.changed
+
+- name: Add PostgreSQL Yum Repository
+  yum:
+    name: '{{ POSGRESQL_RPM_URL }}'
+    state: present
+
+- name: Install PostgreSQL
+  yum: name={{ item }} state=installed
+  with_items:
+  - postgresql10
+  - postgresql10-contrib
+  - postgresql10-server
+  - postgresql10-libs
+
+- name: Install PostgreSQL Python libraries
+  package:
+    name: python-psycopg2
+    state: present
+
+- name: Initialize a database
+  shell: /usr/pgsql-10/bin/postgresql-10-setup initdb
+
+- name: Enable password authentication
+  replace:
+    path: '{{ PG_HBA_PATH }}'
+    regexp: 'ident'
+    replace: "md5"
+
+- name: Start and enable PostgreSQL server
+  service: name=postgresql-10 state=started enabled=yes

roles/elixir/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- import_tasks: ubuntu.yml
+  when: ansible_os_family == "Debian"
+
+- import_tasks: centos.yml
+  when: ansible_os_family == "RedHat"

roles/elixir/tasks/ubuntu.yml

@@ -0,0 +1,51 @@
+---
+
+- name: Add ubuntu/erlang_solutions signing key
+  apt_key:
+    url: '{{ ERLANG_SIGNING_KEY_URL }}'
+    state: present
+
+- name: Add erlang-solutions repository into sources list
+  apt_repository:
+    repo: 'deb {{ ERLANG_SOURCE_DEB }} {{ansible_distribution_release}} contrib'
+    state: present
+
+- name: Install erlang and elixir
+  apt:
+    pkg: "{{ item }}"
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+  with_items:
+  - esl-erlang
+  - elixir
+
+- name: Add PostgreSQL signing key
+  apt_key:
+    url: '{{ POSTGRESQL_SIGNING_KEY_URL }}'
+    state: present
+
+- name: Add PostgreSQL into sources list
+  apt_repository:
+    repo: 'deb {{ POSTGRESQL_SOURCE_DEB }} {{ POSTGRESQL_DISTRIBUTION_RELEASE }} main'
+    state: present
+
+- name: Install PostgreSQL and other tools
+  apt:
+    pkg: "{{ item }}"
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+  with_items:
+  - postgresql
+  - postgresql-contrib
+  - automake
+  - libtool
+  - libpng-dev
+  - zlib1g-dev
+  - inotify-tools
+
+- name: Install python-psycopg2
+  pip:
+    name: psycopg2
+    state: present

site.yml

@@ -17,3 +17,7 @@
 - name: Validator deployment
   import_playbook: validator.yml
   tags: validator
+
+- name: Blockscout deployment
+  import_playbook: blockscout.yml
+  tags: blockscout

tests/blockscout.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "This is a stub for blockscout integration tests"