diff --git a/PENDING.md b/PENDING.md index 4ca3e6df5..70277f24b 100644 --- a/PENDING.md +++ b/PENDING.md @@ -72,6 +72,8 @@ IMPROVEMENTS * [x/gov] Initial governance parameters can now be set in the genesis file * [x/stake] \#1815 Sped up the processing of `EditValidator` txs. * [server] \#1930 Transactions indexer indexes all tags by default. +* [tools] Improved terraform and ansible scripts for infrastructure deployment +* [tools] Added ansible script to enable process core dumps BUG FIXES * \#1666 Add intra-tx counter to the genesis validators diff --git a/networks/remote/ansible/ansible.cfg b/networks/remote/ansible/ansible.cfg deleted file mode 100644 index 045c1ea60..000000000 --- a/networks/remote/ansible/ansible.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[defaults] -retry_files_enabled = False -host_key_checking = False - diff --git a/networks/remote/ansible/increase-openfiles.yml b/networks/remote/ansible/increase-openfiles.yml new file mode 100644 index 000000000..1adcb821c --- /dev/null +++ b/networks/remote/ansible/increase-openfiles.yml @@ -0,0 +1,8 @@ +--- + +- hosts: all + any_errors_fatal: true + gather_facts: no + roles: + - increase-openfiles + diff --git a/networks/remote/ansible/install-datadog-agent.yml b/networks/remote/ansible/install-datadog-agent.yml index 54b77e1f8..b88600eae 100644 --- a/networks/remote/ansible/install-datadog-agent.yml +++ b/networks/remote/ansible/install-datadog-agent.yml @@ -6,6 +6,7 @@ any_errors_fatal: true gather_facts: no roles: + - setup-journald - install-datadog-agent - update-datadog-agent diff --git a/networks/remote/ansible/roles/add-lcd/defaults/main.yml b/networks/remote/ansible/roles/add-lcd/defaults/main.yml index 952d016f7..16a85e0dd 100644 --- a/networks/remote/ansible/roles/add-lcd/defaults/main.yml +++ b/networks/remote/ansible/roles/add-lcd/defaults/main.yml @@ -1,4 +1,4 @@ --- -GAIAD_ADDRESS: tcp://0.0.0.0:1317 +GAIACLI_ADDRESS: tcp://0.0.0.0:1317 diff --git a/networks/remote/ansible/roles/add-lcd/templates/gaiacli.service.j2 b/networks/remote/ansible/roles/add-lcd/templates/gaiacli.service.j2 index 4f189f8f5..a0c20a5b9 100644 --- a/networks/remote/ansible/roles/add-lcd/templates/gaiacli.service.j2 +++ b/networks/remote/ansible/roles/add-lcd/templates/gaiacli.service.j2 @@ -8,7 +8,7 @@ Restart=on-failure User=gaiad Group=gaiad PermissionsStartOnly=true -ExecStart=/usr/bin/gaiacli advanced rest-server --laddr {{GAIAD_ADDRESS}} +ExecStart=/usr/bin/gaiacli advanced rest-server --laddr {{GAIACLI_ADDRESS}} ExecReload=/bin/kill -HUP $MAINPID KillSignal=SIGTERM diff --git a/networks/remote/ansible/roles/increase-openfiles/files/50-fs.conf b/networks/remote/ansible/roles/increase-openfiles/files/50-fs.conf new file mode 100644 index 000000000..5193edd22 --- /dev/null +++ b/networks/remote/ansible/roles/increase-openfiles/files/50-fs.conf @@ -0,0 +1 @@ +fs.file-max=262144 diff --git a/networks/remote/ansible/roles/increase-openfiles/files/91-nofiles.conf b/networks/remote/ansible/roles/increase-openfiles/files/91-nofiles.conf new file mode 100644 index 000000000..929081c6c --- /dev/null +++ b/networks/remote/ansible/roles/increase-openfiles/files/91-nofiles.conf @@ -0,0 +1,3 @@ +* soft nofile 262144 +* hard nofile 262144 + diff --git a/networks/remote/ansible/roles/increase-openfiles/files/limits.conf b/networks/remote/ansible/roles/increase-openfiles/files/limits.conf new file mode 100644 index 000000000..d3fcd2e86 --- /dev/null +++ b/networks/remote/ansible/roles/increase-openfiles/files/limits.conf @@ -0,0 +1,3 @@ +[Service] +LimitNOFILE=infinity +LimitMEMLOCK=infinity diff --git a/networks/remote/ansible/roles/increase-openfiles/handlers/main.yml b/networks/remote/ansible/roles/increase-openfiles/handlers/main.yml new file mode 100644 index 000000000..d49602300 --- /dev/null +++ b/networks/remote/ansible/roles/increase-openfiles/handlers/main.yml @@ -0,0 +1,5 @@ +--- + +- name: reload systemctl + systemd: name=systemd daemon_reload=yes + diff --git a/networks/remote/ansible/roles/increase-openfiles/tasks/main.yml b/networks/remote/ansible/roles/increase-openfiles/tasks/main.yml new file mode 100644 index 000000000..78432f5b5 --- /dev/null +++ b/networks/remote/ansible/roles/increase-openfiles/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# Based on: https://stackoverflow.com/questions/38155108/how-to-increase-limit-for-open-processes-and-files-using-ansible + +- name: Set sysctl File Limits + copy: + src: 50-fs.conf + dest: /etc/sysctl.d + +- name: Set Shell File Limits + copy: + src: 91-nofiles.conf + dest: /etc/security/limits.d + +- name: Set gaia filehandle Limits + copy: + src: limits.conf + dest: "/lib/systemd/system/{{item}}.service.d" + notify: reload systemctl + with_items: + - gaiad + - gaiacli + diff --git a/networks/remote/ansible/roles/install-datadog-agent/files/intake.logs.datadoghq.com.crt b/networks/remote/ansible/roles/install-datadog-agent/files/intake.logs.datadoghq.com.crt deleted file mode 100644 index ef6d9b2c2..000000000 --- a/networks/remote/ansible/roles/install-datadog-agent/files/intake.logs.datadoghq.com.crt +++ /dev/null @@ -1,78 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIESTCCAzGgAwIBAgITBn+UV4WH6Kx33rJTMlu8mYtWDTANBgkqhkiG9w0BAQsF -ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 -b24gUm9vdCBDQSAxMB4XDTE1MTAyMjAwMDAwMFoXDTI1MTAxOTAwMDAwMFowRjEL -MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEVMBMGA1UECxMMU2VydmVyIENB -IDFCMQ8wDQYDVQQDEwZBbWF6b24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDCThZn3c68asg3Wuw6MLAd5tES6BIoSMzoKcG5blPVo+sDORrMd4f2AbnZ -cMzPa43j4wNxhplty6aUKk4T1qe9BOwKFjwK6zmxxLVYo7bHViXsPlJ6qOMpFge5 -blDP+18x+B26A0piiQOuPkfyDyeR4xQghfj66Yo19V+emU3nazfvpFA+ROz6WoVm -B5x+F2pV8xeKNR7u6azDdU5YVX1TawprmxRC1+WsAYmz6qP+z8ArDITC2FMVy2fw -0IjKOtEXc/VfmtTFch5+AfGYMGMqqvJ6LcXiAhqG5TI+Dr0RtM88k+8XUBCeQ8IG -KuANaL7TiItKZYxK1MMuTJtV9IblAgMBAAGjggE7MIIBNzASBgNVHRMBAf8ECDAG -AQH/AgEAMA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUWaRmBlKge5WSPKOUByeW -dFv5PdAwHwYDVR0jBBgwFoAUhBjMhTTsvAyUlC4IWZzHshBOCggwewYIKwYBBQUH -AQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8vb2NzcC5yb290Y2ExLmFtYXpvbnRy -dXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDovL2NydC5yb290Y2ExLmFtYXpvbnRy -dXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8EODA2MDSgMqAwhi5odHRwOi8vY3Js -LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY3JsMBMGA1UdIAQMMAow -CAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IBAQCFkr41u3nPo4FCHOTjY3NTOVI1 -59Gt/a6ZiqyJEi+752+a1U5y6iAwYfmXss2lJwJFqMp2PphKg5625kXg8kP2CN5t -6G7bMQcT8C8xDZNtYTd7WPD8UZiRKAJPBXa30/AbwuZe0GaFEQ8ugcYQgSn+IGBI -8/LwhBNTZTUVEWuCUUBVV18YtbAiPq3yXqMB48Oz+ctBWuZSkbvkNodPLamkB2g1 -upRyzQ7qDn1X8nn8N8V7YJ6y68AtkHcNSRAnpTitxBKjtKPISLMVCx7i4hncxHZS -yLyKQXhw2W2Xs0qLeC1etA+jTGDK4UfLeC0SF7FSi8o5LL21L8IzApar2pR/ ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIEkjCCA3qgAwIBAgITBn+USionzfP6wq4rAfkI7rnExjANBgkqhkiG9w0BAQsF -ADCBmDELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNj -b3R0c2RhbGUxJTAjBgNVBAoTHFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4x -OzA5BgNVBAMTMlN0YXJmaWVsZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1 -dGhvcml0eSAtIEcyMB4XDTE1MDUyNTEyMDAwMFoXDTM3MTIzMTAxMDAwMFowOTEL -MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv -b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj -ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM -9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw -IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 -VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L -93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm -jgSubJrIqg0CAwEAAaOCATEwggEtMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ -BAQDAgGGMB0GA1UdDgQWBBSEGMyFNOy8DJSULghZnMeyEE4KCDAfBgNVHSMEGDAW -gBScXwDfqgHXMCs4iKK4bUqc8hGRgzB4BggrBgEFBQcBAQRsMGowLgYIKwYBBQUH -MAGGImh0dHA6Ly9vY3NwLnJvb3RnMi5hbWF6b250cnVzdC5jb20wOAYIKwYBBQUH -MAKGLGh0dHA6Ly9jcnQucm9vdGcyLmFtYXpvbnRydXN0LmNvbS9yb290ZzIuY2Vy -MD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly9jcmwucm9vdGcyLmFtYXpvbnRydXN0 -LmNvbS9yb290ZzIuY3JsMBEGA1UdIAQKMAgwBgYEVR0gADANBgkqhkiG9w0BAQsF -AAOCAQEAYjdCXLwQtT6LLOkMm2xF4gcAevnFWAu5CIw+7bMlPLVvUOTNNWqnkzSW -MiGpSESrnO09tKpzbeR/FoCJbM8oAxiDR3mjEH4wW6w7sGDgd9QIpuEdfF7Au/ma -eyKdpwAJfqxGF4PcnCZXmTA5YpaP7dreqsXMGz7KQ2hsVxa81Q4gLv7/wmpdLqBK -bRRYh5TmOTFffHPLkIhqhBGWJ6bt2YFGpn6jcgAKUj6DiAdjd4lpFw85hdKrCEVN -0FE6/V1dN2RMfjCyVSRCnTawXZwXgWHxyvkQAiSr6w10kY17RSlQOYiypok1JR4U -akcjMS9cmvqtmg5iUaQqqcT5NJ0hGA== ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIEdTCCA12gAwIBAgIJAKcOSkw0grd/MA0GCSqGSIb3DQEBCwUAMGgxCzAJBgNV -BAYTAlVTMSUwIwYDVQQKExxTdGFyZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTIw -MAYDVQQLEylTdGFyZmllbGQgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 -eTAeFw0wOTA5MDIwMDAwMDBaFw0zNDA2MjgxNzM5MTZaMIGYMQswCQYDVQQGEwJV -UzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTElMCMGA1UE -ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjE7MDkGA1UEAxMyU3RhcmZp -ZWxkIFNlcnZpY2VzIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEi -MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVDDrEKvlO4vW+GZdfjohTsR8/ -y8+fIBNtKTrID30892t2OGPZNmCom15cAICyL1l/9of5JUOG52kbUpqQ4XHj2C0N -Tm/2yEnZtvMaVq4rtnQU68/7JuMauh2WLmo7WJSJR1b/JaCTcFOD2oR0FMNnngRo -Ot+OQFodSk7PQ5E751bWAHDLUu57fa4657wx+UX2wmDPE1kCK4DMNEffud6QZW0C -zyyRpqbn3oUYSXxmTqM6bam17jQuug0DuDPfR+uxa40l2ZvOgdFFRjKWcIfeAg5J -Q4W2bHO7ZOphQazJ1FTfhy/HIrImzJ9ZVGif/L4qL8RVHHVAYBeFAlU5i38FAgMB -AAGjgfAwge0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0O -BBYEFJxfAN+qAdcwKziIorhtSpzyEZGDMB8GA1UdIwQYMBaAFL9ft9HO3R+G9FtV -rNzXEMIOqYjnME8GCCsGAQUFBwEBBEMwQTAcBggrBgEFBQcwAYYQaHR0cDovL28u -c3MyLnVzLzAhBggrBgEFBQcwAoYVaHR0cDovL3guc3MyLnVzL3guY2VyMCYGA1Ud -HwQfMB0wG6AZoBeGFWh0dHA6Ly9zLnNzMi51cy9yLmNybDARBgNVHSAECjAIMAYG -BFUdIAAwDQYJKoZIhvcNAQELBQADggEBACMd44pXyn3pF3lM8R5V/cxTbj5HD9/G -VfKyBDbtgB9TxF00KGu+x1X8Z+rLP3+QsjPNG1gQggL4+C/1E2DUBc7xgQjB3ad1 -l08YuW3e95ORCLp+QCztweq7dp4zBncdDQh/U90bZKuCJ/Fp1U1ervShw3WnWEQt -8jxwmKy6abaVd38PMV4s/KCHOkdp8Hlf9BRUpJVeEXgSYCfOn8J3/yNTd126/+pZ -59vPr5KW7ySaNRB6nJHGDn2Z9j8Z3/VyVOEVqQdZe4O/Ui5GjLIAZHYcSNPYeehu -VsyuLAOQ1xk4meTKCRlb/weWsKh/NEnfVqn3sF/tM+2MR7cwA130A4w= ------END CERTIFICATE----- diff --git a/networks/remote/ansible/roles/install-datadog-agent/files/logrotate.conf b/networks/remote/ansible/roles/install-datadog-agent/files/logrotate.conf deleted file mode 100644 index e90a5ffb2..000000000 --- a/networks/remote/ansible/roles/install-datadog-agent/files/logrotate.conf +++ /dev/null @@ -1,35 +0,0 @@ -# see "man logrotate" for details -# rotate log files weekly -daily - -# keep 4 days worth of backlogs -rotate 4 - -# create new (empty) log files after rotating old ones -create - -# use date as a suffix of the rotated file -dateext - -# uncomment this if you want your log files compressed -compress - -# RPM packages drop log rotation information into this directory -include /etc/logrotate.d - -# no packages own wtmp and btmp -- we'll rotate them here -/var/log/wtmp { - monthly - create 0664 root utmp - minsize 1M - rotate 1 -} - -/var/log/btmp { - missingok - monthly - create 0600 root utmp - rotate 1 -} - -# system-specific logs may be also be configured here. diff --git a/networks/remote/ansible/roles/install-datadog-agent/files/syslog b/networks/remote/ansible/roles/install-datadog-agent/files/syslog deleted file mode 100644 index 8052df477..000000000 --- a/networks/remote/ansible/roles/install-datadog-agent/files/syslog +++ /dev/null @@ -1,13 +0,0 @@ -/var/log/cron -/var/log/maillog -/var/log/messages -/var/log/secure -/var/log/spooler -{ - missingok - sharedscripts - postrotate - /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true - service datadog-agent restart 2> /dev/null || true - endscript -} diff --git a/networks/remote/ansible/roles/install-datadog-agent/tasks/main.yml b/networks/remote/ansible/roles/install-datadog-agent/tasks/main.yml index 0c0a49eca..4d5aa1877 100644 --- a/networks/remote/ansible/roles/install-datadog-agent/tasks/main.yml +++ b/networks/remote/ansible/roles/install-datadog-agent/tasks/main.yml @@ -13,38 +13,3 @@ DD_API_KEY: "{{DD_API_KEY}}" DD_HOST_TAGS: "testnet:{{TESTNET_NAME}},cluster:{{CLUSTER_NAME}}" -- name: Disable journald rate-limiting - lineinfile: "dest=/etc/systemd/journald.conf regexp={{item.regexp}} line='{{item.line}}'" - with_items: - - { regexp: "^#RateLimitInterval", line: "RateLimitInterval=0s" } - - { regexp: "^#RateLimitBurst", line: "RateLimitBurst=0" } - - { regexp: "^#SystemMaxFileSize", line: "SystemMaxFileSize=500M" } - notify: restart journald - -- name: As long as Datadog does not support journald on RPM-based linux, we enable rsyslog - yum: "name={{item}} state=installed" - with_items: - - rsyslog - - rsyslog-gnutls - -#- name: Get DataDog certificate for rsyslog -# get_url: url=https://docs.datadoghq.com/crt/intake.logs.datadoghq.com.crt dest=/etc/ssl/certs/intake.logs.datadoghq.com.crt - -- name: Get DataDog certificate for rsyslog - copy: src=intake.logs.datadoghq.com.crt dest=/etc/ssl/certs/intake.logs.datadoghq.com.crt - -- name: Add datadog config to rsyslog - template: src=datadog.conf.j2 dest=/etc/rsyslog.d/datadog.conf mode=0600 - notify: restart rsyslog - -- name: Set logrotate to rotate daily so syslog does not use up all space - copy: src=logrotate.conf dest=/etc/logrotate.conf - -- name: Set syslog to restart datadog-agent after logrotate - copy: src=syslog dest=/etc/logrotate.d/syslog - -#semanage port -a -t syslog_tls_port_t -p tcp 10516 -- name: Enable rsyslog to report to port 10516 in SELinux - seport: ports=10516 proto=tcp reload=yes setype=syslog_tls_port_t state=present - notify: restart rsyslog - diff --git a/networks/remote/ansible/roles/install-datadog-agent/templates/datadog.conf.j2 b/networks/remote/ansible/roles/install-datadog-agent/templates/datadog.conf.j2 deleted file mode 100644 index 1ab7d1b07..000000000 --- a/networks/remote/ansible/roles/install-datadog-agent/templates/datadog.conf.j2 +++ /dev/null @@ -1,14 +0,0 @@ -$template DatadogFormat,"{{DD_API_KEY}} <%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% - - - %msg%\n" - -$imjournalRatelimitInterval 0 -$imjournalRatelimitBurst 0 - -$DefaultNetstreamDriver gtls -$DefaultNetstreamDriverCAFile /etc/ssl/certs/intake.logs.datadoghq.com.crt -$ActionSendStreamDriver gtls -$ActionSendStreamDriverMode 1 -$ActionSendStreamDriverAuthMode x509/name -$ActionSendStreamDriverPermittedPeer *.logs.datadoghq.com -*.* @@intake.logs.datadoghq.com:10516;DatadogFormat - - diff --git a/networks/remote/ansible/roles/set-debug/files/sysconfig/gaiacli b/networks/remote/ansible/roles/set-debug/files/sysconfig/gaiacli new file mode 100644 index 000000000..8ef3a7e0c --- /dev/null +++ b/networks/remote/ansible/roles/set-debug/files/sysconfig/gaiacli @@ -0,0 +1 @@ +DAEMON_COREFILE_LIMIT='unlimited' diff --git a/networks/remote/ansible/roles/set-debug/files/sysconfig/gaiad b/networks/remote/ansible/roles/set-debug/files/sysconfig/gaiad new file mode 100644 index 000000000..8ef3a7e0c --- /dev/null +++ b/networks/remote/ansible/roles/set-debug/files/sysconfig/gaiad @@ -0,0 +1 @@ +DAEMON_COREFILE_LIMIT='unlimited' diff --git a/networks/remote/ansible/roles/set-debug/files/sysctl.d/10-procdump b/networks/remote/ansible/roles/set-debug/files/sysctl.d/10-procdump new file mode 100644 index 000000000..fbbbe0512 --- /dev/null +++ b/networks/remote/ansible/roles/set-debug/files/sysctl.d/10-procdump @@ -0,0 +1,3 @@ +kernel.core_uses_pid = 1 +kernel.core_pattern = /tmp/core-%e-%s-%u-%g-%p-%t +fs.suid_dumpable = 2 diff --git a/networks/remote/ansible/roles/set-debug/handlers/main.yaml b/networks/remote/ansible/roles/set-debug/handlers/main.yaml new file mode 100644 index 000000000..743ce09bc --- /dev/null +++ b/networks/remote/ansible/roles/set-debug/handlers/main.yaml @@ -0,0 +1,4 @@ +--- + +- name: reload sysctl + command: "/sbin/sysctl -p" diff --git a/networks/remote/ansible/roles/set-debug/tasks/main.yml b/networks/remote/ansible/roles/set-debug/tasks/main.yml new file mode 100644 index 000000000..7497dabd8 --- /dev/null +++ b/networks/remote/ansible/roles/set-debug/tasks/main.yml @@ -0,0 +1,9 @@ +--- +# Based on https://www.cyberciti.biz/tips/linux-core-dumps.html + +- name: Copy sysctl and sysconfig files to enable app and daemon core dumps + file: src=. dest=/etc/ + notify: reload sysctl + +- name: Enable debugging for all apps + lineinfile: create=yes line="DAEMON_COREFILE_LIMIT='unlimited'" path=/etc/sysconfig/init regexp=^DAEMON_COREFILE_LIMIT= diff --git a/networks/remote/terraform-app/files/gaiad.service b/networks/remote/ansible/roles/setup-fullnodes/files/gaiad.service similarity index 100% rename from networks/remote/terraform-app/files/gaiad.service rename to networks/remote/ansible/roles/setup-fullnodes/files/gaiad.service diff --git a/networks/remote/ansible/roles/setup-fullnodes/handlers/main.yml b/networks/remote/ansible/roles/setup-fullnodes/handlers/main.yml new file mode 100644 index 000000000..987e2947b --- /dev/null +++ b/networks/remote/ansible/roles/setup-fullnodes/handlers/main.yml @@ -0,0 +1,5 @@ +--- + +- name: reload systemd + systemd: name=gaiad enabled=yes daemon_reload=yes + diff --git a/networks/remote/ansible/roles/setup-fullnodes/tasks/main.yml b/networks/remote/ansible/roles/setup-fullnodes/tasks/main.yml index 26bcc4ccd..ba9b22942 100644 --- a/networks/remote/ansible/roles/setup-fullnodes/tasks/main.yml +++ b/networks/remote/ansible/roles/setup-fullnodes/tasks/main.yml @@ -6,14 +6,21 @@ run_once: true become: no +- name: Create gaiad user + user: name=gaiad home=/home/gaiad shell=/bin/bash + - name: Copy binary copy: src: "{{BINARY}}" dest: /usr/bin mode: 0755 +- name: Copy service file + copy: src=gaiad.service dest=/etc/systemd/system/gaiad.service mode=0755 + notify: reload systemd + - name: Get node ID - command: "cat /etc/gaiad-nodeid" + command: "cat /etc/nodeid" changed_when: false register: nodeid diff --git a/networks/remote/ansible/roles/setup-journald/tasks/main.yml b/networks/remote/ansible/roles/setup-journald/tasks/main.yml index a7a105bf3..130da5200 100644 --- a/networks/remote/ansible/roles/setup-journald/tasks/main.yml +++ b/networks/remote/ansible/roles/setup-journald/tasks/main.yml @@ -5,8 +5,17 @@ with_items: - { regexp: "^#RateLimitInterval", line: "RateLimitInterval=0s" } - { regexp: "^#RateLimitBurst", line: "RateLimitBurst=0" } + - { regexp: "^#SystemMaxFileSize", line: "SystemMaxFileSize=100M" } + - { regexp: "^#SystemMaxUse", line: "SystemMaxUse=500M" } + - { regexp: "^#SystemMaxFiles", line: "SystemMaxFiles=10" } notify: restart journald +- name: Change logrotate to daily + lineinfile: "dest=/etc/logrotate.conf regexp={{item.regexp}} line='{{item.line}}'" + with_items: + - { regexp: "^weekly", line: "daily" } + - { regexp: "^#compress", line: "compress" } + - name: Create journal directory for permanent logs file: path=/var/log/journal state=directory notify: restart journald diff --git a/networks/remote/terraform-aws/files/gaiad.service b/networks/remote/ansible/roles/setup-validators/files/gaiad.service similarity index 100% rename from networks/remote/terraform-aws/files/gaiad.service rename to networks/remote/ansible/roles/setup-validators/files/gaiad.service diff --git a/networks/remote/ansible/roles/setup-validators/handlers/main.yml b/networks/remote/ansible/roles/setup-validators/handlers/main.yml new file mode 100644 index 000000000..987e2947b --- /dev/null +++ b/networks/remote/ansible/roles/setup-validators/handlers/main.yml @@ -0,0 +1,5 @@ +--- + +- name: reload systemd + systemd: name=gaiad enabled=yes daemon_reload=yes + diff --git a/networks/remote/ansible/roles/setup-validators/tasks/main.yml b/networks/remote/ansible/roles/setup-validators/tasks/main.yml index 46e9117d5..8832db4aa 100644 --- a/networks/remote/ansible/roles/setup-validators/tasks/main.yml +++ b/networks/remote/ansible/roles/setup-validators/tasks/main.yml @@ -6,14 +6,21 @@ run_once: true become: no +- name: Create gaiad user + user: name=gaiad home=/home/gaiad shell=/bin/bash + - name: Copy binary copy: src: "{{BINARY}}" dest: /usr/bin mode: 0755 +- name: Copy service file + copy: src=gaiad.service dest=/etc/systemd/system/gaiad.service mode=0755 + notify: reload systemd + - name: Get node ID - command: "cat /etc/gaiad-nodeid" + command: "cat /etc/nodeid" changed_when: false register: nodeid diff --git a/networks/remote/ansible/roles/update-datadog-agent/templates/datadog.yaml.j2 b/networks/remote/ansible/roles/update-datadog-agent/templates/datadog.yaml.j2 index 2f3eb286e..7e35a5255 100644 --- a/networks/remote/ansible/roles/update-datadog-agent/templates/datadog.yaml.j2 +++ b/networks/remote/ansible/roles/update-datadog-agent/templates/datadog.yaml.j2 @@ -28,10 +28,10 @@ api_key: {{DD_API_KEY}} # Setting this option to "yes" will force the agent to only use TLS 1.2 when # pushing data to the url specified in "dd_url". -# force_tls_12: no +force_tls_12: yes # Force the hostname to whatever you want. (default: auto-detected) -# hostname: mymachine.mydomain +hostname: {{inventory_hostname}} # Make the agent use "hostname -f" on unix-based systems as a last resort # way of determining the hostname instead of Golang "os.Hostname()" @@ -220,7 +220,7 @@ collect_ec2_tags: true # Logs agent # # Logs agent is disabled by default -logs_enabled: true +#logs_enabled: true # # Enable logs collection for all containers, disabled by default # logs_config: diff --git a/networks/remote/ansible/set-corefilesize.yml b/networks/remote/ansible/set-corefilesize.yml deleted file mode 100644 index ae0f85291..000000000 --- a/networks/remote/ansible/set-corefilesize.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -# Set the core file size to unlimited to allow the system to generate core dumps - -- hosts: all - any_errors_fatal: true - gather_facts: no - - tasks: - - - name: Set core file size to unlimited to be able to get the core dump on SIGABRT - shell: "ulimit -c unlimited" - diff --git a/networks/remote/ansible/set-debug.yml b/networks/remote/ansible/set-debug.yml new file mode 100644 index 000000000..76ee1b357 --- /dev/null +++ b/networks/remote/ansible/set-debug.yml @@ -0,0 +1,8 @@ +--- + +- hosts: all + any_errors_fatal: true + gather_facts: no + roles: + - set-debug + diff --git a/networks/remote/ansible/setup-fullnodes.yml b/networks/remote/ansible/setup-fullnodes.yml index 7175e4d36..da1810d1d 100644 --- a/networks/remote/ansible/setup-fullnodes.yml +++ b/networks/remote/ansible/setup-fullnodes.yml @@ -2,10 +2,12 @@ #GENESISFILE required #CONFIGFILE required +#BINARY required - hosts: all any_errors_fatal: true gather_facts: no roles: + - increase-openfiles - setup-fullnodes diff --git a/networks/remote/ansible/setup-validators.yml b/networks/remote/ansible/setup-validators.yml index b8cec9386..0e6f2959a 100644 --- a/networks/remote/ansible/setup-validators.yml +++ b/networks/remote/ansible/setup-validators.yml @@ -4,5 +4,6 @@ any_errors_fatal: true gather_facts: no roles: + - increase-openfiles - setup-validators diff --git a/networks/remote/ansible/status.yml b/networks/remote/ansible/status.yml index d0b89d13f..ebd7f72ee 100644 --- a/networks/remote/ansible/status.yml +++ b/networks/remote/ansible/status.yml @@ -9,7 +9,7 @@ - name: Gather status uri: body_format: json - url: "http://{{inventory_hostname}}:26657/status" + url: "http://{{ansible_host}}:26657/status" register: status - name: Print status diff --git a/networks/remote/terraform-app/files/terraform.sh b/networks/remote/terraform-app/files/terraform.sh index 754c5757f..60b4dd8e7 100644 --- a/networks/remote/terraform-app/files/terraform.sh +++ b/networks/remote/terraform-app/files/terraform.sh @@ -4,13 +4,5 @@ #Usage: terraform.sh #Add gaiad node number for remote identification -echo "$2" > /etc/gaiad-nodeid - -#Create gaiad user -useradd -m -s /bin/bash gaiad - -#Reload services to enable the gaiad service (note that the gaiad binary is not available yet) -systemctl daemon-reload -systemctl enable gaiad - +echo "$2" > /etc/nodeid diff --git a/networks/remote/terraform-app/infra/attachment.tf b/networks/remote/terraform-app/infra/attachment.tf index daf9e6faa..1ba5f4fe5 100644 --- a/networks/remote/terraform-app/infra/attachment.tf +++ b/networks/remote/terraform-app/infra/attachment.tf @@ -6,10 +6,16 @@ #Instance Attachment (autoscaling is the future) resource "aws_lb_target_group_attachment" "lb_attach" { - count = "${var.SERVERS*length(data.aws_availability_zones.zones.names)}" + count = "${var.SERVERS*min(length(data.aws_availability_zones.zones.names),var.max_zones)}" target_group_arn = "${aws_lb_target_group.lb_target_group.arn}" target_id = "${element(aws_instance.node.*.id,count.index)}" - port = 80 + port = 26657 } +resource "aws_lb_target_group_attachment" "lb_attach_lcd" { + count = "${var.SERVERS*min(length(data.aws_availability_zones.zones.names),var.max_zones)}" + target_group_arn = "${aws_lb_target_group.lb_target_group_lcd.arn}" + target_id = "${element(aws_instance.node.*.id,count.index)}" + port = 1317 +} diff --git a/networks/remote/terraform-app/infra/instance.tf b/networks/remote/terraform-app/infra/instance.tf index c2ccd5d44..53b21e62d 100644 --- a/networks/remote/terraform-app/infra/instance.tf +++ b/networks/remote/terraform-app/infra/instance.tf @@ -13,7 +13,7 @@ data "aws_ami" "linux" { resource "aws_instance" "node" { # depends_on = ["${element(aws_route_table_association.route_table_association.*,count.index)}"] - count = "${var.SERVERS*length(data.aws_availability_zones.zones.names)}" + count = "${var.SERVERS*min(length(data.aws_availability_zones.zones.names),var.max_zones)}" ami = "${data.aws_ami.linux.image_id}" instance_type = "${var.instance_type}" key_name = "${aws_key_pair.key.key_name}" @@ -33,7 +33,7 @@ resource "aws_instance" "node" { } root_block_device { - volume_size = 20 + volume_size = 40 } connection { @@ -47,14 +47,8 @@ resource "aws_instance" "node" { destination = "/tmp/terraform.sh" } - provisioner "file" { - source = "files/gaiad.service" - destination = "/tmp/gaiad.service" - } - provisioner "remote-exec" { inline = [ - "sudo cp /tmp/gaiad.service /etc/systemd/system/gaiad.service", "chmod +x /tmp/terraform.sh", "sudo /tmp/terraform.sh ${var.name} ${count.index}", ] diff --git a/networks/remote/terraform-app/infra/lb.tf b/networks/remote/terraform-app/infra/lb.tf index b4f6f120c..201a53ffd 100644 --- a/networks/remote/terraform-app/infra/lb.tf +++ b/networks/remote/terraform-app/infra/lb.tf @@ -1,20 +1,22 @@ resource "aws_lb" "lb" { name = "${var.name}" subnets = ["${aws_subnet.subnet.*.id}"] -# security_groups = ["${split(",", var.lb_security_groups)}"] + security_groups = ["${aws_security_group.secgroup.id}"] tags { Name = "${var.name}" } # access_logs { # bucket = "${var.s3_bucket}" -# prefix = "ELB-logs" +# prefix = "lblogs" # } } resource "aws_lb_listener" "lb_listener" { load_balancer_arn = "${aws_lb.lb.arn}" - port = "80" - protocol = "HTTP" + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS-1-2-Ext-2018-06" + certificate_arn = "${var.certificate_arn}" default_action { target_group_arn = "${aws_lb_target_group.lb_target_group.arn}" @@ -23,7 +25,6 @@ resource "aws_lb_listener" "lb_listener" { } resource "aws_lb_listener_rule" "listener_rule" { -# depends_on = ["aws_lb_target_group.lb_target_group"] listener_arn = "${aws_lb_listener.lb_listener.arn}" priority = "100" action { @@ -38,24 +39,14 @@ resource "aws_lb_listener_rule" "listener_rule" { resource "aws_lb_target_group" "lb_target_group" { name = "${var.name}" - port = "80" + port = "26657" protocol = "HTTP" vpc_id = "${aws_vpc.vpc.id}" tags { name = "${var.name}" } -# stickiness { -# type = "lb_cookie" -# cookie_duration = 1800 -# enabled = "true" -# } -# health_check { -# healthy_threshold = 3 -# unhealthy_threshold = 10 -# timeout = 5 -# interval = 10 -# path = "${var.target_group_path}" -# port = "${var.target_group_port}" -# } + health_check { + path = "/health" + } } diff --git a/networks/remote/terraform-app/infra/lcd.tf b/networks/remote/terraform-app/infra/lcd.tf new file mode 100644 index 000000000..5d09903d0 --- /dev/null +++ b/networks/remote/terraform-app/infra/lcd.tf @@ -0,0 +1,39 @@ +resource "aws_lb_listener" "lb_listener_lcd" { + load_balancer_arn = "${aws_lb.lb.arn}" + port = "1317" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS-1-2-Ext-2018-06" + certificate_arn = "${var.certificate_arn}" + + default_action { + target_group_arn = "${aws_lb_target_group.lb_target_group_lcd.arn}" + type = "forward" + } +} + +resource "aws_lb_listener_rule" "listener_rule_lcd" { + listener_arn = "${aws_lb_listener.lb_listener_lcd.arn}" + priority = "100" + action { + type = "forward" + target_group_arn = "${aws_lb_target_group.lb_target_group_lcd.id}" + } + condition { + field = "path-pattern" + values = ["/"] + } +} + +resource "aws_lb_target_group" "lb_target_group_lcd" { + name = "${var.name}lcd" + port = "1317" + protocol = "HTTP" + vpc_id = "${aws_vpc.vpc.id}" + tags { + name = "${var.name}" + } + health_check { + path = "/node_version" + } +} + diff --git a/networks/remote/terraform-app/infra/outputs.tf b/networks/remote/terraform-app/infra/outputs.tf index 525cb0d31..fdb32611c 100644 --- a/networks/remote/terraform-app/infra/outputs.tf +++ b/networks/remote/terraform-app/infra/outputs.tf @@ -8,9 +8,9 @@ output "instances" { value = ["${aws_instance.node.*.id}"] } -output "instances_count" { - value = "${length(aws_instance.node.*)}" -} +#output "instances_count" { +# value = "${length(aws_instance.node.*)}" +#} // The list of cluster instance public IPs output "public_ips" { diff --git a/networks/remote/terraform-app/infra/variables.tf b/networks/remote/terraform-app/infra/variables.tf index 8459e78f0..0a96f1443 100644 --- a/networks/remote/terraform-app/infra/variables.tf +++ b/networks/remote/terraform-app/infra/variables.tf @@ -17,6 +17,11 @@ variable "SERVERS" { default = "1" } +variable "max_zones" { + description = "Maximum number of availability zones to use" + default = "1" +} + variable "ssh_private_file" { description = "SSH private key file to be used to connect to the nodes" type = "string" @@ -27,3 +32,8 @@ variable "ssh_public_file" { type = "string" } +variable "certificate_arn" { + description = "Load-balancer SSL certificate AWS ARN" + type = "string" +} + diff --git a/networks/remote/terraform-app/infra/vpc.tf b/networks/remote/terraform-app/infra/vpc.tf index b38d845ca..638ccfe0b 100644 --- a/networks/remote/terraform-app/infra/vpc.tf +++ b/networks/remote/terraform-app/infra/vpc.tf @@ -33,7 +33,7 @@ data "aws_availability_zones" "zones" { } resource "aws_subnet" "subnet" { - count = "${length(data.aws_availability_zones.zones.names)}" + count = "${min(length(data.aws_availability_zones.zones.names),var.max_zones)}" vpc_id = "${aws_vpc.vpc.id}" availability_zone = "${element(data.aws_availability_zones.zones.names,count.index)}" cidr_block = "${cidrsubnet(aws_vpc.vpc.cidr_block, 8, count.index)}" @@ -45,7 +45,7 @@ resource "aws_subnet" "subnet" { } resource "aws_route_table_association" "route_table_association" { - count = "${length(data.aws_availability_zones.zones.names)}" + count = "${min(length(data.aws_availability_zones.zones.names),var.max_zones)}" subnet_id = "${element(aws_subnet.subnet.*.id,count.index)}" route_table_id = "${aws_route_table.route_table.id}" } @@ -66,8 +66,15 @@ resource "aws_security_group" "secgroup" { } ingress { - from_port = 80 - to_port = 80 + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 1317 + to_port = 1317 protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] } diff --git a/networks/remote/terraform-app/main.tf b/networks/remote/terraform-app/main.tf index de828daf0..687e3b5b7 100644 --- a/networks/remote/terraform-app/main.tf +++ b/networks/remote/terraform-app/main.tf @@ -9,11 +9,16 @@ variable "SERVERS" { default = "1" } +variable "MAX_ZONES" { + description = "Maximum number of availability zones to use" + default = "4" +} + #See https://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region #eu-west-3 does not contain CentOS images variable "REGION" { description = "AWS Regions" - default = "us-east-2" + default = "us-east-1" } variable "SSH_PRIVATE_FILE" { @@ -26,6 +31,11 @@ variable "SSH_PUBLIC_FILE" { type = "string" } +variable "CERTIFICATE_ARN" { + description = "Load-balancer certificate AWS ARN" + type = "string" +} + # ap-southeast-1 and ap-southeast-2 does not contain the newer CentOS 1704 image variable "image" { description = "AWS image name" @@ -34,7 +44,7 @@ variable "image" { variable "instance_type" { description = "AWS instance type" - default = "t2.medium" + default = "t2.large" } provider "aws" { @@ -48,7 +58,9 @@ module "nodes" { instance_type = "${var.instance_type}" ssh_public_file = "${var.SSH_PUBLIC_FILE}" ssh_private_file = "${var.SSH_PRIVATE_FILE}" + certificate_arn = "${var.CERTIFICATE_ARN}" SERVERS = "${var.SERVERS}" + max_zones = "${var.MAX_ZONES}" } output "public_ips" { diff --git a/networks/remote/terraform-aws/files/terraform.sh b/networks/remote/terraform-aws/files/terraform.sh index ef8019972..47363b37d 100644 --- a/networks/remote/terraform-aws/files/terraform.sh +++ b/networks/remote/terraform-aws/files/terraform.sh @@ -7,13 +7,5 @@ REGION="$(($2 + 1))" RNODE="$(($3 + 1))" ID="$((${REGION} * 100 + ${RNODE}))" -echo "$ID" > /etc/gaiad-nodeid - -#Create gaiad user -useradd -m -s /bin/bash gaiad - -#Reload services to enable the gaiad service (note that the gaiad binary is not available yet) -systemctl daemon-reload -systemctl enable gaiad - +echo "$ID" > /etc/nodeid diff --git a/networks/remote/terraform-aws/main.tf b/networks/remote/terraform-aws/main.tf index 448695389..41e05995e 100644 --- a/networks/remote/terraform-aws/main.tf +++ b/networks/remote/terraform-aws/main.tf @@ -43,7 +43,7 @@ variable "image" { variable "instance_type" { description = "AWS instance type" - default = "t2.medium" + default = "t2.large" } module "nodes-0" { diff --git a/networks/remote/terraform-aws/nodes/main.tf b/networks/remote/terraform-aws/nodes/main.tf index 854f7ac2a..825be4af6 100644 --- a/networks/remote/terraform-aws/nodes/main.tf +++ b/networks/remote/terraform-aws/nodes/main.tf @@ -79,7 +79,7 @@ resource "aws_instance" "node" { } root_block_device { - volume_size = 20 + volume_size = 40 } connection { @@ -93,14 +93,8 @@ resource "aws_instance" "node" { destination = "/tmp/terraform.sh" } - provisioner "file" { - source = "files/gaiad.service" - destination = "/tmp/gaiad.service" - } - provisioner "remote-exec" { inline = [ - "sudo cp /tmp/gaiad.service /etc/systemd/system/gaiad.service", "chmod +x /tmp/terraform.sh", "sudo /tmp/terraform.sh ${var.name} ${var.multiplier} ${count.index}", ] diff --git a/networks/remote/terraform-do/cluster/main.tf b/networks/remote/terraform-do/cluster/main.tf index 9bada5915..07331ff3d 100644 --- a/networks/remote/terraform-do/cluster/main.tf +++ b/networks/remote/terraform-do/cluster/main.tf @@ -29,11 +29,6 @@ resource "digitalocean_droplet" "cluster" { destination = "/tmp/terraform.sh" } - provisioner "file" { - source = "files/gaiad.service" - destination = "/etc/systemd/system/gaiad.service" - } - provisioner "remote-exec" { inline = [ "chmod +x /tmp/terraform.sh", diff --git a/networks/remote/terraform-do/files/gaiad.service b/networks/remote/terraform-do/files/gaiad.service deleted file mode 100644 index 697166567..000000000 --- a/networks/remote/terraform-do/files/gaiad.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=gaiad -Requires=network-online.target -After=network-online.target - -[Service] -Restart=on-failure -User=gaiad -Group=gaiad -PermissionsStartOnly=true -ExecStart=/usr/bin/gaiad start -ExecReload=/bin/kill -HUP $MAINPID -KillSignal=SIGTERM - -[Install] -WantedBy=multi-user.target - diff --git a/networks/remote/terraform-do/files/terraform.sh b/networks/remote/terraform-do/files/terraform.sh index 39d89ea82..60b4dd8e7 100644 --- a/networks/remote/terraform-do/files/terraform.sh +++ b/networks/remote/terraform-do/files/terraform.sh @@ -4,16 +4,5 @@ #Usage: terraform.sh #Add gaiad node number for remote identification -echo "$2" > /etc/gaiad-nodeid - -#Create gaiad user -useradd -m -s /bin/bash gaiad -#cp -r /root/.ssh /home/gaiad/.ssh -#chown -R gaiad.gaiad /home/gaiad/.ssh -#chmod -R 700 /home/gaiad/.ssh - -#Reload services to enable the gaiad service (note that the gaiad binary is not available yet) -systemctl daemon-reload -systemctl enable gaiad - +echo "$2" > /etc/nodeid