From e500858b2e27173ac6433ca9f03c1e6c8eb2e2c9 Mon Sep 17 00:00:00 2001 From: Pavel Khakhulin Date: Thu, 12 Oct 2017 22:25:11 +0300 Subject: [PATCH] Add templates for owner --- README.md | 4 + TestTestNet/mining-node/install.sh | 2 +- TestTestNet/owner/install.sh | 350 +++++++++++++++++++++++++++++ TestTestNet/owner/node.toml | 34 +++ TestTestNet/owner/template.json | 253 +++++++++++++++++++++ 5 files changed, 642 insertions(+), 1 deletion(-) create mode 100644 TestTestNet/owner/install.sh create mode 100644 TestTestNet/owner/node.toml create mode 100644 TestTestNet/owner/template.json diff --git a/README.md b/README.md index b272980..0b33863 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,8 @@ ### Netstats server +[![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Foraclesorg%2Ftest-templates%2Fdev-mainnet%2FTestTestNet%2Fowner%2Ftemplate.json) + +### Netstats server + [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Foraclesorg%2Ftest-templates%2Fdev-mainnet%2FTestTestNet%2Fnetstats-server%2Ftemplate.json) diff --git a/TestTestNet/mining-node/install.sh b/TestTestNet/mining-node/install.sh index 11a5a89..31e8376 100644 --- a/TestTestNet/mining-node/install.sh +++ b/TestTestNet/mining-node/install.sh @@ -182,8 +182,8 @@ EOF } start_pm2_via_systemd() { - sudo npm install pm2 -g echo "=====> start_pm2_via_systemd" + sudo npm install pm2 -g sudo bash -c "cat > /etc/systemd/system/oracles-pm2.service < start_logentries" + sudo bash -c "echo 'deb http://rep.logentries.com/ trusty main' > /etc/apt/sources.list.d/logentries.list" + sudo bash -c "gpg --keyserver pgp.mit.edu --recv-keys C43C79AD && gpg -a --export C43C79AD | apt-key add -" + sudo apt-get update + sudo apt-get install -y logentries + sudo le reinit --user-key=0665901a-e843-41c5-82c1-2cc4b39f0b21 --pull-server-side-config=False + + mkdir -p /home/${ADMIN_USERNAME}/logs + touch /home/${ADMIN_USERNAME}/logs/parity.log + touch /home/${ADMIN_USERNAME}/logs/parity.err + touch /home/${ADMIN_USERNAME}/logs/parity.out + + sudo bash -c "cat >> /etc/le/config << EOF +[install_err] +path = /var/lib/waagent/custom-script/download/0/stderr +destination = dev-mainnet/${EXT_IP} + +[install_out] +path = /var/lib/waagent/custom-script/download/0/stdout +destination = dev-mainnet/${EXT_IP} + +[parity_log] +path = /home/${ADMIN_USERNAME}/logs/parity.log +destination = dev-mainnet/${EXT_IP} + +[netstats_daemon_err] +path = /home/${ADMIN_USERNAME}/logs/netstats_daemon.err +destination = dev-mainnet/${EXT_IP} + +[netstats_daemon_out] +path = /home/${ADMIN_USERNAME}/logs/netstats_daemon.out +destination = dev-mainnet/${EXT_IP} + +[parity_err] +path = /home/${ADMIN_USERNAME}/logs/parity.err +destination = dev-mainnet/${EXT_IP} + +[parity_out] +path = /home/${ADMIN_USERNAME}/logs/parity.out +destination = dev-mainnet/${EXT_IP} + +EOF" + sudo apt-get install -y logentries-daemon + sudo service logentries start + echo "<===== start_logentries" +} + +start_logentries + +# */ + +echo "========== dev-mainnet/owner/install.sh starting ==========" +echo "===== current time: $(date)" +echo "===== username: $(whoami)" +echo "===== working directory: $(pwd)" +echo "===== operating system info:" +lsb_release -a +echo "===== memory usage info:" +free -m +echo "===== external ip: ${EXT_IP}" + +echo "===== environmental variables:" +printenv + +INSTALL_CONFIG_REPO="https://raw.githubusercontent.com/oraclesorg/test-templates/dev-mainnet/TestTestNet/owner" +GENESIS_REPO_LOC="https://raw.githubusercontent.com/oraclesorg/oracles-scripts/master/spec.json" +GENESIS_JSON="spec.json" +NODE_TOML="node.toml" +NODE_PWD="node.pwd" +BOOTNODES_TXT="https://raw.githubusercontent.com/oraclesorg/test-templates/dev-mainnet/TestTestNet/bootnodes.txt" + +export HOME="${HOME:-/home/${ADMIN_USERNAME}}" + +echo "===== repo base path: ${INSTALL_CONFIG_REPO}" + +# this should be provided through env by azure template +NETSTATS_SERVER="${NETSTATS_SERVER}" +NETSTATS_SECRET="${NETSTATS_SECRET}" +OWNER_KEYFILE="${OWNER_KEYFILE}" +OWNER_KEYPASS="${OWNER_KEYPASS}" +NODE_FULLNAME="${NODE_FULLNAME:-Owner}" +NODE_ADMIN_EMAIL="${NODE_ADMIN_EMAIL:-somebody@somehere}" +ADMIN_USERNAME="${ADMIN_USERNAME}" + +prepare_homedir() { + echo "=====> prepare_homedir" + # ln -s "$(pwd)" "/home/${ADMIN_USERNAME}/script-dir" + cd "/home/${ADMIN_USERNAME}" + echo "Now changed directory to: $(pwd)" + mkdir -p logs + mkdir -p logs/old + echo "<===== prepare_homedir" +} + +install_ntpd() { + echo "=====> install_ntpd" + sudo timedatectl set-ntp no + sudo apt-get -y install ntp + + sudo bash -c "cat > /etc/cron.hourly/ntpdate << EOF +#!/bin/sh +sudo service ntp stop +sudo ntpdate -s ntp.ubuntu.com +sudo service ntp start +EOF" + sudo chmod 755 /etc/cron.hourly/ntpdate + echo "<===== install_ntpd" +} + +install_haveged() { + echo "=====> install_haveged" + sudo apt-get -y install haveged + sudo update-rc.d haveged defaults + echo "<===== install_haveged" +} + +allocate_swap() { + echo "=====> allocate_swap" + sudo apt-get -y install bc + #sudo fallocate -l $(echo "$(free -b | awk '/Mem/{ print $2 }')*2" | bc -l) /swapfile + sudo fallocate -l 1G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + sudo sh -c "printf '/swapfile none swap sw 0 0\n' >> /etc/fstab" + sudo sh -c "printf 'vm.swappiness=10\n' >> /etc/sysctl.conf" + sudo sysctl vm.vfs_cache_pressure=50 + sudo sh -c "printf 'vm.vfs_cache_pressure = 50\n' >> /etc/sysctl.conf" + echo "<===== allocate_swap" +} + +pull_image_and_configs() { + echo "=====> pull_image_and_configs" + # curl -s -O "${INSTALL_CONFIG_REPO}/../${GENESIS_JSON}" + curl -s -o "${GENESIS_JSON}" "${GENESIS_REPO_LOC}" + curl -s -O "${INSTALL_CONFIG_REPO}/${NODE_TOML}" + curl -s -o "bootnodes.txt" "${BOOTNODES_TXT}" + sed -i "/\[network\]/a nat=\"extip:${EXT_IP}\"" ${NODE_TOML} + sed -i "/\[network\]/a bootnodes=\[$(cat bootnodes.txt | sed 's/\r$//' | awk -F'#' '{ print $1 }' | awk '/enode/{ print "\""$1"\"" }' | paste -sd "," -)\]" ${NODE_TOML} + cat >> ${NODE_TOML} < "${NODE_PWD}" + mkdir -p parity/keys/OraclesPoA + echo ${OWNER_KEYFILE} | base64 -d > parity/keys/OraclesPoA/owner.key + + echo "<===== pull_image_and_configs" +} + +clone_dapps() { + echo "=====> clone_dapps" + mkdir -p parity/dapps + git clone https://github.com/oraclesorg/oracles-dapps-keys-generation.git parity/dapps/KeysGenerator + git clone https://github.com/oraclesorg/oracles-dapps-voting.git parity/dapps/Voting + git clone https://github.com/oraclesorg/oracles-dapps-validators.git parity/dapps/ValidatorsList + echo "<===== clone_dapps" +} + +install_nodejs() { + echo "=====> install_nodejs" + # curl -sL https://deb.nodesource.com/setup_0.12 | bash - + curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - + sudo apt-get update + sudo apt-get install -y build-essential git unzip wget nodejs ntp cloud-utils + + # add symlink if it doesn't exist + [[ ! -f /usr/bin/node ]] && sudo ln -s /usr/bin/nodejs /usr/bin/node + echo "<===== install_nodejs" +} + +start_pm2_via_systemd() { + echo "=====> start_pm2_via_systemd" + sudo npm install pm2 -g + sudo bash -c "cat > /etc/systemd/system/oracles-pm2.service < install_netstats_via_systemd" + git clone https://github.com/oraclesorg/eth-net-intelligence-api + cd eth-net-intelligence-api + #sed -i '/"web3"/c "web3": "0.19.x",' package.json + npm install + sudo npm install pm2 -g + + cat > app.json << EOL +[ + { + "name" : "netstats_daemon", + "script" : "app.js", + "log_date_format" : "YYYY-MM-DD HH:mm:SS Z", + "error_file" : "/home/${ADMIN_USERNAME}/logs/netstats_daemon.err", + "out_file" : "/home/${ADMIN_USERNAME}/logs/netstats_daemon.out", + "merge_logs" : false, + "watch" : false, + "max_restarts" : 100, + "exec_interpreter" : "node", + "exec_mode" : "fork_mode", + "env": + { + "NODE_ENV" : "production", + "RPC_HOST" : "localhost", + "RPC_PORT" : "8545", + "LISTENING_PORT" : "30300", + "INSTANCE_NAME" : "${NODE_FULLNAME}", + "CONTACT_DETAILS" : "${NODE_ADMIN_EMAIL}", + "WS_SERVER" : "http://${NETSTATS_SERVER}:3000", + "WS_SECRET" : "${NETSTATS_SECRET}", + "VERBOSITY" : 2 + } + } +] +EOL + cd .. + sudo bash -c "cat > /etc/systemd/system/oracles-netstats.service < use_deb_via_systemd" + curl -LO 'http://parity-downloads-mirror.parity.io/v1.7.0/x86_64-unknown-linux-gnu/parity_1.7.0_amd64.deb' + sudo dpkg -i parity_1.7.0_amd64.deb + + #curl -LO 'http://d1h4xl4cr1h0mo.cloudfront.net/nightly/x86_64-unknown-debian-gnu/parity_1.8.0_amd64.deb' + #sudo dpkg -i parity_1.8.0_amd64.deb + + sudo bash -c "cat > /etc/systemd/system/oracles-parity.service < configure_logrotate" + + sudo bash -c "cat > /etc/logrotate.d/oracles.conf << EOF +/home/${ADMIN_USERNAME}/logs/*.log { + rotate 10 + size 200M + missingok + compress + copytruncate + dateext + dateformat %Y-%m-%d-%s + olddir old +} +/home/${ADMIN_USERNAME}/.pm2/pm2.log { + su ${ADMIN_USERNAME} ${ADMIN_USERNAME} + rotate 10 + size 200M + missingok + compress + copytruncate + dateext + dateformat %Y-%m-%d-%s +} +EOF" + echo "<===== configure_logrotate" +} + +download_initial_keys_script() { + echo "=====> download_initial_keys_script" + git clone https://github.com/oraclesorg/oracles-initial-keys + cd oracles-initial-keys + npm install + cd .. + echo "<===== download_initial_keys_script" +} + +# MAIN +main () { + sudo apt-get update + + prepare_homedir + + install_ntpd + install_haveged + allocate_swap + + install_nodejs + pull_image_and_configs + clone_dapps + + use_deb_via_systemd + + start_pm2_via_systemd + install_netstats_via_systemd + + configure_logrotate + + download_initial_keys_script +} + +main +echo "========== dev-mainnet/owner/install.sh finished ==========" diff --git a/TestTestNet/owner/node.toml b/TestTestNet/owner/node.toml new file mode 100644 index 0000000..b05573a --- /dev/null +++ b/TestTestNet/owner/node.toml @@ -0,0 +1,34 @@ +# dev-mainnet branch +[parity] +chain = "spec.json" +base_path = "parity" +[network] +port = 30300 +discovery=true +[rpc] +cors = "all" +interface = "all" +hosts = ["all"] +port = 8545 +apis = ["web3", "eth", "net", "personal", "parity", "parity_set", "traces", "rpc", "parity_accounts"] +[ui] +interface = "0.0.0.0" +port = 8180 +[websockets] +disable = false +port = 8546 +interface = "all" +origins = ["all"] +apis = ["web3", "eth", "net", "parity", "traces", "rpc", "secretstore"] +hosts = ["all"] +[dapps] +cors = "all" +interface = "0.0.0.0" +hosts = ["all"] +port = 8080 +[account] +password = ["node.pwd"] +[mining] +force_sealing = true +engine_signer = "0xdd0bb0e2a1594240fed0c2f2c17c1e9ab4f87126" +reseal_on_txs = "none" diff --git a/TestTestNet/owner/template.json b/TestTestNet/owner/template.json new file mode 100644 index 0000000..fb0e307 --- /dev/null +++ b/TestTestNet/owner/template.json @@ -0,0 +1,253 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "nodeFullName": { + "type": "string", + "metadata": { + "description": "Node full name (publicly visible)" + } + }, + "nodeAdminEmail": { + "type": "string", + "metadata": { + "description": "Node admin email (publicly visible)" + } + }, + "ownerKeyFile": { + "type": "secureString", + "metadata": { + "description": "Content of the file with mining key (json string)" + } + }, + "ownerKeypass": { + "type": "secureString", + "metadata": { + "description": "Password for the mining key" + } + }, + "adminUsername": { + "type": "string", + "metadata": { + "description": "Username for the Virtual Machine" + } + }, + "sshPublicKey": { + "type": "string", + "metadata": { + "description": "SSH public key for the Virtual Machine" + } + }, + "netstatsServer": { + "type": "string", + "metadata": { + "description": "NetStats server IP" + } + }, + "netstatsSecret": { + "type": "secureString", + "metadata": { + "description": "NetStats net code" + } + } + }, + "variables": { + "storageAccountName": "[concat(uniquestring(resourceGroup().id), 'salinuxvm')]", + "imagePublisher": "Canonical", + "imageOffer": "UbuntuServer", + "nicName": "myVMNic", + "addressPrefix": "10.0.0.0/16", + "subnetName": "Subnet", + "subnetPrefix": "10.0.0.0/24", + "storageAccountType": "Standard_LRS", + "publicIPAddressName": "myPublicIP", + "publicIPAddressType": "Static", + "vmName": "MyUbuntuVM", + "vmSize": "Standard_A1", + "virtualNetworkName": "MyVNET", + "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", + "subnetRef": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]", + "ubuntuOSVersion": "16.04.0-LTS", + "dnsLabelPrefix": "[concat('oracles-bn-', uniqueString(resourceGroup().id, parameters('nodeFullName')))]", + "adminUsername": "[trim(toLower(parameters('adminUsername')))]", + "sshPublicKey": "[trim(parameters('sshPublicKey'))]", + "nodeFullName": "[trim(parameters('nodeFullName'))]", + "nodeAdminEmail": "[trim(parameters('nodeAdminEmail'))]", + "ownerKeyFile": "[base64(trim(string(parameters('ownerKeyFile'))))]", + "ownerKeypass": "[trim(parameters('ownerKeypass'))]", + "netstatsServer": "[trim(parameters('netstatsServer'))]", + "netstatsSecret": "[trim(parameters('netstatsSecret'))]" + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "[variables('storageAccountName')]", + "apiVersion": "2017-06-01", + "location": "[resourceGroup().location]", + "sku": { + "name": "[variables('storageAccountType')]" + }, + "kind": "Storage", + "properties": {} + }, + { + "apiVersion": "2017-04-01", + "type": "Microsoft.Network/publicIPAddresses", + "name": "[variables('publicIPAddressName')]", + "location": "[resourceGroup().location]", + "properties": { + "publicIPAllocationMethod": "[variables('publicIPAddressType')]", + "dnsSettings": { + "domainNameLabel": "[variables('dnsLabelPrefix')]" + } + } + }, + { + "apiVersion": "2017-04-01", + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('virtualNetworkName')]", + "location": "[resourceGroup().location]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('addressPrefix')]" + ] + }, + "subnets": [ + { + "name": "[variables('subnetName')]", + "properties": { + "addressPrefix": "[variables('subnetPrefix')]" + } + } + ] + } + }, + { + "apiVersion": "2017-04-01", + "type": "Microsoft.Network/networkInterfaces", + "name": "[variables('nicName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]", + "[resourceId('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]" + }, + "subnet": { + "id": "[variables('subnetRef')]" + } + } + } + ] + } + }, + { + "apiVersion": "2017-03-30", + "type": "Microsoft.Compute/virtualMachines", + "name": "[variables('vmName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]", + "[resourceId('Microsoft.Network/networkInterfaces/', variables('nicName'))]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[variables('vmSize')]" + }, + "osProfile": { + "computerName": "[variables('vmName')]", + "adminUsername": "[variables('adminUsername')]", + "adminPassword": "", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[concat('/home/', variables('adminUsername'), '/.ssh/authorized_keys')]", + "keyData": "[variables('sshPublicKey')]" + } + ] + } + } + }, + "storageProfile": { + "imageReference": { + "publisher": "[variables('imagePublisher')]", + "offer": "[variables('imageOffer')]", + "sku": "[variables('ubuntuOSVersion')]", + "version": "latest" + }, + "osDisk": { + "createOption": "FromImage" + }, + "dataDisks": [ + { + "diskSizeGB": "128", + "lun": 0, + "createOption": "Empty" + } + ] + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces',variables('nicName'))]" + } + ] + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "enabled": "true", + "storageUri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName')), '2016-01-01').primaryEndpoints.blob)]" + } + } + }, + "resources": [ + { + "apiVersion": "2015-06-15", + "type": "extensions", + "name": "config-app", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', variables('vmName'))]" + ], + "tags": { + "displayName": "config-app" + }, + "properties": { + "publisher": "Microsoft.Azure.Extensions", + "type": "CustomScript", + "typeHandlerVersion": "2.0", + "autoUpgradeMinorVersion": true, + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/oraclesorg/test-templates/dev/TestTestNet/bootnode/install.sh" + ] + }, + "protectedSettings": { + "commandToExecute": "[concat('cp install.sh /home/', variables('adminUsername'), '/; cd /home/', variables('adminUsername'), '; chown ', variables('adminUsername'), ':', variables('adminUsername'), ' install.sh; chmod +x install.sh; ADMIN_USERNAME=\"', variables('adminUsername'), '\" NETSTATS_SERVER=\"', variables('netstatsServer'), '\" NETSTATS_SECRET=\"', variables('netstatsSecret'), '\" OWNER_KEYFILE=\"', variables('ownerKeyFile'), '\" OWNER_KEYPASS=\"', variables('ownerKeypass'), '\" NODE_FULLNAME=\"', variables('nodeFullName'), '\" NODE_ADMIN_EMAIL=\"', variables('nodeAdminEmail'), '\" sudo -u ', variables('adminUsername'), ' -E -H ./install.sh')]" + } + } + } + ] + } + ], + "outputs": { + "hostname": { + "type": "string", + "value": "[reference(variables('publicIPAddressName')).dnsSettings.fqdn]" + }, + "sshCommand": { + "type": "string", + "value": "[concat('ssh ', variables('adminUsername'), '@', reference(variables('publicIPAddressName')).dnsSettings.fqdn)]" + } + } +}