diff --git a/.flake8 b/.flake8 index 920e652a..ea38e9cb 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,4 @@ [flake8] max-line-length = 120 exclude = ./typings/**/* +ignore = E203,W503 diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fae60ef8..79dee7a0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,7 +20,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - brew update && brew install gmp boost || echo "" + brew update && brew install gmp boost openssl || echo "" + sudo apt-get install libboost-all-dev || echo "" + sudo apt-get install libssl-dev || echo "" sh install.sh - name: Test proof of space run: | @@ -38,7 +40,7 @@ jobs: cd ../../../ - name: Lint source with flake8 run: | - ./.venv/bin/flake8 src --exclude src/wallet + ./.venv/bin/flake8 src --exclude src/wallet/electron/node_modules - name: Lint source with mypy run: | ./.venv/bin/mypy src tests diff --git a/.gitignore b/.gitignore index fadb6636..74461410 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,8 @@ pip-delete-this-directory.txt # Packaging chia-blockchain.tar.gz -src/wallet/electron/node_modules/ \ No newline at end of file +# Electron wallet node modules +src/wallet/electron/node_modules/ + +# Bip158 build dir +lib/bip158/build/ \ No newline at end of file diff --git a/INSTALL.md b/INSTALL.md index 616f90d2..cde1a258 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ To install the chia-blockchain node, follow the instructions according to your o Make sure [brew](https://brew.sh/) is available before starting the setup. ```bash brew upgrade python -brew install cmake gmp +brew install cmake gmp boost openssl git clone https://github.com/Chia-Network/chia-blockchain.git cd chia-blockchain diff --git a/README.md b/README.md index 558fef23..c49074bc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,203 @@ For alpha testnet most should only install harvesters, farmers, plotter and full To install the chia-blockchain node, follow [these](INSTALL.md) instructions according to your operating system. -## Step 2: Generate keys +```bash +sudo apt-get update +sudo apt-get install build-essential git cmake libgmp3-dev --no-install-recommends +sudo apt-get install python3-dev python3-venv --no-install-recommends + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +. .venv/bin/activate +``` +### Amazon Linux 2 + +```bash +sudo yum update +sudo yum install gcc-c++ cmake3 wget git openssl openssl-devel +sudo yum install python3 python3-devel libffi-devel gmp-devel + +# CMake - add a symlink for cmake3 - required by blspy +sudo ln -s /usr/bin/cmake3 /usr/local/bin/cmake + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +. .venv/bin/activate +``` +### CentOS 7.7 + +```bash +sudo yum update +sudo yum install centos-release-scl-rh epel-release +sudo yum install devtoolset-8-toolchain cmake3 libffi-devel +sudo yum install gmp-devel libsqlite3x-devel +sudo yum install wget git openssl openssl-devel + +# CMake - add a symlink for cmake3 - required by blspy +sudo ln -s /usr/bin/cmake3 /usr/local/bin/cmake + +scl enable devtoolset-8 bash + +# Install Python 3.7.5 (current rpm's are 3.6.x) +wget https://www.python.org/ftp/python/3.7.5/Python-3.7.5.tgz +tar -zxvf Python-3.7.5.tgz; cd Python-3.7.5 +./configure --enable-optimizations; sudo make install; cd .. + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +. .venv/bin/activate +``` +### RHEL 8.1 + +```bash +sudo yum update +sudo yum install gcc-c++ cmake3 git openssl openssl-devel +sudo yum install wget make libffi-devel gmp-devel sqlite-devel + +# Install Python 3.7.5 (current rpm's are 3.6.x) +wget https://www.python.org/ftp/python/3.7.5/Python-3.7.5.tgz +tar -zxvf Python-3.7.5.tgz; cd Python-3.7.5 +./configure --enable-optimizations; sudo make install; cd .. + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +. .venv/bin/activate +``` +### Windows (WSL + Ubuntu) +#### Install WSL + Ubuntu 18.04 LTS, upgrade to Ubuntu 19.x + +This will require multiple reboots. From an Administrator PowerShell +`Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux` +and then +`Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform`. +Once that is complete, install Ubuntu 18.04 LTS from the Windows Store. +```bash +# Upgrade to 19.x +sudo nano /etc/update-manager/release-upgrades +# Change "Prompt=lts" to "Prompt=normal" save and exit + +sudo apt-get -y update +sudo apt-get -y upgrade +sudo do-release-upgrade + +sudo apt-get install -y build-essential cmake python3-dev python3-venv software-properties-common libgmp3-dev --no-install-recommends + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sudo sh install.sh + +. .venv/bin/activate +``` + +#### Alternate method for Ubuntu 18.04 LTS +In `./install.sh`: +Change `python3` to `python3.7` +Each line that starts with `pip ...` becomes `python3.7 -m pip ...` + +```bash +sudo apt-get -y update +sudo apt-get install -y build-essential cmake python3-dev python3-venv software-properties-common libgmp3-dev --no-install-recommends + +# Install python3.7 with ppa +sudo add-apt-repository -y ppa:deadsnakes/ppa +sudo apt-get -y update +sudo apt-get install -y python3.7 python3.7-venv python3.7-dev + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sudo sh install.sh +. .venv/bin/activate +``` + +### MacOS +Make sure [brew](https://brew.sh/) is available before starting the setup. +```bash +brew upgrade python +brew install cmake gmp + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh +. .venv/bin/activate +``` + + +## Step 2: Install timelord (optional) +Note: this step is needed only if you intend to run a timelord or a local simulation. +These assume you've already successfully installed harvester, farmer, plotting, and full node above. boost 1.66 or newer is required on all platforms. +### Ubuntu/Debian +```bash +cd chia-blockchain + +sh install_timelord.sh +``` +### Amazon Linux 2 and CentOS 7.7 +```bash +#Only for Amazon Linux 2 +sudo amazon-linux-extras install epel + +sudo yum install mpfr-devel + +# Install Boost 1.72.0 +wget https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz +tar -zxvf boost_1_72_0.tar.gz +cd boost_1_72_0 +./bootstrap.sh --prefix=/usr/local +sudo ./b2 install --prefix=/usr/local --with=all; cd .. +LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +cd chia-blockchain + +sh install_timelord.sh +``` +### RHEL 8.1 +```bash +sudo yum install mpfr-devel boost boost-devel + +cd chia-blockchain + +sh install_timelord.sh +``` +### Windows (WSL + Ubuntu) +#### Install WSL + Ubuntu upgraded to 19.x +```bash +cd chia-blockchain + +sh install_timelord.sh +``` +#### Alternate method for Ubuntu 18.04 +```bash +# Install boost 1.70 with ppa +sudo add-apt-repository -y ppa:mhier/libboost-latest +sudo apt-get update +sudo apt-get install libboost1.70 libboost1.70-dev +``` + +### MacOS +```bash +brew install boost + +cd chia-blockchain + +sh install_timelord.sh +``` + +## Step 3: Generate keys First, create some keys by running the following script: ```bash python -m scripts.regenerate_keys @@ -85,5 +281,3 @@ You can also use the [HTTP RPC](https://github.com/Chia-Network/chia-blockchain/ ```bash curl -X POST http://localhost:8555/get_blockchain_state ``` - -After installing, follow the remaining instructions in [README.md](README.md) to run the software. diff --git a/lib/bip158/CMakeLists.txt b/lib/bip158/CMakeLists.txt index 70f85ac5..9effa44c 100644 --- a/lib/bip158/CMakeLists.txt +++ b/lib/bip158/CMakeLists.txt @@ -8,20 +8,23 @@ ENDIF() project(chiabip158) +link_directories(/usr/local/opt/openssl/lib) + include_directories( ${INCLUDE_DIRECTORIES} ${CMAKE_CURRENT_SOURCE_DIR}/src /usr/local/opt/openssl/include + /usr/local/opt/boost/include ) set (CMAKE_CXX_FLAGS "-DHAVE_WORKING_BOOST_SLEEP -g -O3 -Wall -msse2 -msse -march=native -std=c++14 -maes") FILE(GLOB_RECURSE MyCSources src/*.cpp) -ADD_LIBRARY(biplib ${MyCSources}) +ADD_LIBRARY(biplib ${MyCSources}) add_subdirectory(lib/pybind11) -pybind11_add_module(chiabip158 +pybind11_add_module(chiabip158 ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiabip158.cpp ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/PyBIP158.cpp) @@ -29,6 +32,7 @@ add_executable(bip158 main.cpp ) -target_link_libraries(bip158 biplib -lboost_system -lpthread -lboost_thread -lboost_filesystem -lssl -lcrypto) -target_link_libraries(chiabip158 PRIVATE biplib -lboost_system -lpthread -lboost_thread -lboost_filesystem -lssl -lcrypto) +find_package(Boost COMPONENTS system filesystem thread REQUIRED) +target_link_libraries(bip158 biplib ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} -lpthread -lssl -lcrypto) +target_link_libraries(chiabip158 PRIVATE biplib ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} -lpthread -lssl -lcrypto) diff --git a/lib/bip158/python-bindings/PyBIP158.cpp b/lib/bip158/python-bindings/PyBIP158.cpp index c04dc609..826c6401 100644 --- a/lib/bip158/python-bindings/PyBIP158.cpp +++ b/lib/bip158/python-bindings/PyBIP158.cpp @@ -29,6 +29,16 @@ PyBIP158::PyBIP158(std::vector< std::vector< unsigned char > >& hashes) filter=new GCSFilter({0, 0, 20, 1 << 20},elements); } +PyBIP158::PyBIP158(std::vector< unsigned char > & encoded_filter) +{ + filter=new GCSFilter({0, 0, 20, 1 << 20}, encoded_filter); +} + +const std::vector& PyBIP158::GetEncoded() +{ + return filter->GetEncoded(); +} + PyBIP158::~PyBIP158() { delete filter; diff --git a/lib/bip158/python-bindings/PyBIP158.h b/lib/bip158/python-bindings/PyBIP158.h index db41a91c..baaf4bec 100644 --- a/lib/bip158/python-bindings/PyBIP158.h +++ b/lib/bip158/python-bindings/PyBIP158.h @@ -25,6 +25,8 @@ public: public: PyBIP158(std::vector< std::vector< unsigned char > >& hashes); + PyBIP158(std::vector< unsigned char > & encoded_filter); + const std::vector& GetEncoded(); ~PyBIP158(); bool Match(std::vector< unsigned char >& hash); diff --git a/lib/bip158/python-bindings/chiabip158.cpp b/lib/bip158/python-bindings/chiabip158.cpp index 70b67fc5..a43b034a 100644 --- a/lib/bip158/python-bindings/chiabip158.cpp +++ b/lib/bip158/python-bindings/chiabip158.cpp @@ -24,6 +24,8 @@ PYBIND11_MODULE(chiabip158, mod) { py::class_> clsPyBIP158(mod, "PyBIP158"); clsPyBIP158.def(py::init >&>()); + clsPyBIP158.def(py::init< std::vector< unsigned char > &>()); + clsPyBIP158.def("GetEncoded",(const std::vector< unsigned char >& (PyBIP158::*)()) &PyBIP158::GetEncoded); clsPyBIP158.def("Match", (bool (PyBIP158::*)(std::vector< unsigned char >&)) &PyBIP158::Match); clsPyBIP158.def("MatchAny", (bool (PyBIP158::*)(std::vector< std::vector< unsigned char > >&)) &PyBIP158::MatchAny); } diff --git a/lib/chiapos/CMakeLists.txt b/lib/chiapos/CMakeLists.txt index 4a1d1c1d..c45cf137 100644 --- a/lib/chiapos/CMakeLists.txt +++ b/lib/chiapos/CMakeLists.txt @@ -32,6 +32,20 @@ add_subdirectory(lib/pybind11) pybind11_add_module(chiapos ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiapos.cpp) set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++1z -maes") +try_run(CMAKE_AESNI_TEST_RUN_RESULT + CMAKE_AESNI_TEST_COMPILE_RESULT + ${CMAKE_CURRENT_BINARY_DIR}/cmake_aesni_test + ${CMAKE_CURRENT_SOURCE_DIR}/src/cmake_aesni_test.cpp) + +# Did compilation succeed and process return 0 (success)? +IF("${CMAKE_AESNI_TEST_COMPILE_RESULT}" AND ("${CMAKE_AESNI_TEST_RUN_RESULT}" EQUAL 0)) + message(STATUS "AESNI Enabled") + set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++17 -maes") +ELSE() + message(STATUS "AESNI Disabled") + add_compile_definitions (DISABLE_AESNI) + set (CMAKE_CXX_FLAGS "-g -O3 -Wall -march=native -std=c++17") +ENDIF() add_executable(ProofOfSpace src/cli.cpp diff --git a/lib/chiapos/src/aes.hpp b/lib/chiapos/src/aes.hpp index def86c3b..db91ad7c 100644 --- a/lib/chiapos/src/aes.hpp +++ b/lib/chiapos/src/aes.hpp @@ -1,270 +1,493 @@ -// Copyright 2018 Chia Network Inc - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Some public domain code is taken from pycrypto: -// https://github.com/dlitz/pycrypto/blob/master/src/AESNI.c -// -// AESNI.c: AES using AES-NI instructions -// -// Written in 2013 by Sebastian Ramacher - -#ifndef SRC_CPP_AES_HPP_ -#define SRC_CPP_AES_HPP_ - -#include // for memcmp -#include // for intrinsics for AES-NI - -/** - * Encrypts a message of 128 bits with a 128 bit key, using - * 10 rounds of AES128 (9 full rounds and one final round). Uses AES-NI - * assembly instructions. - */ -#define DO_ENC_BLOCK_128(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[0]); \ - m = _mm_aesenc_si128(m, k[1]); \ - m = _mm_aesenc_si128(m, k[2]); \ - m = _mm_aesenc_si128(m, k[3]); \ - m = _mm_aesenc_si128(m, k[4]); \ - m = _mm_aesenc_si128(m, k[5]); \ - m = _mm_aesenc_si128(m, k[6]); \ - m = _mm_aesenc_si128(m, k[7]); \ - m = _mm_aesenc_si128(m, k[8]); \ - m = _mm_aesenc_si128(m, k[9]); \ - m = _mm_aesenclast_si128(m, k[10]); \ - } while (0) - -/** - * Encrypts a message of 128 bits with a 256 bit key, using - * 13 rounds of AES256 (13 full rounds and one final round). Uses - * AES-NI assembly instructions. - */ -#define DO_ENC_BLOCK_256(m, k) \ - do {\ - m = _mm_xor_si128(m, k[ 0]); \ - m = _mm_aesenc_si128(m, k[ 1]); \ - m = _mm_aesenc_si128(m, k[ 2]); \ - m = _mm_aesenc_si128(m, k[ 3]); \ - m = _mm_aesenc_si128(m, k[ 4]); \ - m = _mm_aesenc_si128(m, k[ 5]); \ - m = _mm_aesenc_si128(m, k[ 6]); \ - m = _mm_aesenc_si128(m, k[ 7]); \ - m = _mm_aesenc_si128(m, k[ 8]); \ - m = _mm_aesenc_si128(m, k[ 9]); \ - m = _mm_aesenc_si128(m, k[ 10]);\ - m = _mm_aesenc_si128(m, k[ 11]);\ - m = _mm_aesenc_si128(m, k[ 12]);\ - m = _mm_aesenc_si128(m, k[ 13]);\ - m = _mm_aesenclast_si128(m, k[ 14]);\ - }while(0) - -/** - * Encrypts a message of 128 bits with a 128 bit key, using - * 2 full rounds of AES128. Uses AES-NI assembly instructions. - */ -#define DO_ENC_BLOCK_2ROUND(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[0]); \ - m = _mm_aesenc_si128(m, k[1]); \ - m = _mm_aesenc_si128(m, k[2]); \ - } while (0) -/** - * Decrypts a ciphertext of 128 bits with a 128 bit key, using - * 10 rounds of AES128 (9 full rounds and one final round). - * Uses AES-NI assembly instructions. - */ -#define DO_DEC_BLOCK(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[10 + 0]); \ - m = _mm_aesdec_si128(m, k[10 + 1]); \ - m = _mm_aesdec_si128(m, k[10 + 2]); \ - m = _mm_aesdec_si128(m, k[10 + 3]); \ - m = _mm_aesdec_si128(m, k[10 + 4]); \ - m = _mm_aesdec_si128(m, k[10 + 5]); \ - m = _mm_aesdec_si128(m, k[10 + 6]); \ - m = _mm_aesdec_si128(m, k[10 + 7]); \ - m = _mm_aesdec_si128(m, k[10 + 8]); \ - m = _mm_aesdec_si128(m, k[10 + 9]); \ - m = _mm_aesdeclast_si128(m, k[0]); \ - } while (0) - -/** - * Decrypts a ciphertext of 128 bits with a 128 bit key, using - * 2 full rounds of AES128. Uses AES-NI assembly instructions. - * Will not work unless key schedule is modified. - */ /* -#define DO_DEC_BLOCK_2ROUND(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[2 + 0]); \ - m = _mm_aesdec_si128(m, k[2 + 1]); \ - m = _mm_aesdec_si128(m, k[2 + 2]); \ - } while (0) + +The code in this file is originally from the Tiny AES project, which is in the +public domain. + +https://github.com/kokke/tiny-AES-c + +It has been heavily modified by Chia. + +*** + +This is an implementation of the AES algorithm, specifically ECB, CTR and CBC mode. +Block size can be chosen in aes.h - available choices are AES128, AES192, AES256. + +The implementation is verified against the test vectors in: + National Institute of Standards and Technology Special Publication 800-38A 2001 ED + +ECB-AES128 +---------- + + plain-text: + 6bc1bee22e409f96e93d7e117393172a + ae2d8a571e03ac9c9eb76fac45af8e51 + 30c81c46a35ce411e5fbc1191a0a52ef + f69f2445df4f9b17ad2b417be66c3710 + + key: + 2b7e151628aed2a6abf7158809cf4f3c + + resulting cipher + 3ad77bb40d7a3660a89ecaf32466ef97 + f5d3d58503b9699de785895a96fdbaaf + 43b1cd7f598ece23881b00e3ed030688 + 7b0c785e27e8ad3f8223207104725dd4 + + +NOTE: String length must be evenly divisible by 16byte (str_len % 16 == 0) + You should pad the end of the string with zeros if this is not the case. + For AES192/256 the key size is proportionally larger. + */ -static __m128i key_schedule[20]; // The expanded key -static __m128i aes128_keyexpand(__m128i key) { - key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); - key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); - return _mm_xor_si128(key, _mm_slli_si128(key, 4)); +/*****************************************************************************/ +/* Includes: */ +/*****************************************************************************/ +#include +#include // CBC mode, for memset + +//#define DISABLE_AESNI + +#ifndef DISABLE_AESNI +#include +#include "aesni.hpp" + +bool bHasAES=false; +bool bCheckedAES=false; +#endif // DISABLE_AESNI + +/*****************************************************************************/ +/* Defines: */ +/*****************************************************************************/ +// The number of columns comprising a state in AES. This is a constant in AES. Value=4 +#define Nb 4 + +/*****************************************************************************/ +/* Private variables: */ +/*****************************************************************************/ +// state - array holding the intermediate results during decryption. +typedef uint8_t state_t[4][4]; + +// The lookup-tables are marked const so they can be placed in read-only storage instead of RAM +// The numbers below can be computed dynamically trading ROM for RAM - +// This can be useful in (embedded) bootloader applications, where ROM is often limited. +static const uint8_t sbox[256] = { + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; + +static const uint8_t rsbox[256] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d }; + +// The round constant word array, Rcon[i], contains the values given by +// x to the power (i-1) being powers of x (x is denoted as {02}) in the field GF(2^8) +static const uint8_t Rcon[11] = { + 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; + +/* + * Jordan Goulder points out in PR #12 (https://github.com/kokke/tiny-AES-C/pull/12), + * that you can remove most of the elements in the Rcon array, because they are unused. + * + * From Wikipedia's article on the Rijndael key schedule @ https://en.wikipedia.org/wiki/Rijndael_key_schedule#Rcon + * + * "Only the first some of these constants are actually used – up to rcon[10] for AES-128 (as 11 round keys are needed), + * up to rcon[8] for AES-192, up to rcon[7] for AES-256. rcon[0] is not used in AES algorithm." + */ + +/*****************************************************************************/ +/* Private functions: */ +/*****************************************************************************/ + +#define getSBoxValue(num) (sbox[(num)]) + + +// This function adds the round key to state. +// The round key is added to the state by an XOR function. +static void AddRoundKey(uint8_t round, state_t* state, const uint8_t* RoundKey) +{ + uint8_t i,j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j]; + } + } } -#define KEYEXP128_H(K1, K2, I, S) _mm_xor_si128(aes128_keyexpand(K1), \ - _mm_shuffle_epi32(_mm_aeskeygenassist_si128(K2, I), S)) +// The SubBytes Function Substitutes the values in the +// state matrix with values in an S-box. +static void SubBytes(state_t* state) +{ + uint8_t i, j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[j][i] = getSBoxValue((*state)[j][i]); + } + } +} -#define KEYEXP128(K, I) KEYEXP128_H(K, K, I, 0xff) -#define KEYEXP256(K1, K2, I) KEYEXP128_H(K1, K2, I, 0xff) -#define KEYEXP256_2(K1, K2) KEYEXP128_H(K1, K2, 0x00, 0xaa) +// The ShiftRows() function shifts the rows in the state to the left. +// Each row is shifted with different offset. +// Offset = Row number. So the first row is not shifted. +static void ShiftRows(state_t* state) +{ + uint8_t temp; -// public API + // Rotate first row 1 columns to left + temp = (*state)[0][1]; + (*state)[0][1] = (*state)[1][1]; + (*state)[1][1] = (*state)[2][1]; + (*state)[2][1] = (*state)[3][1]; + (*state)[3][1] = temp; + + // Rotate second row 2 columns to left + temp = (*state)[0][2]; + (*state)[0][2] = (*state)[2][2]; + (*state)[2][2] = temp; + + temp = (*state)[1][2]; + (*state)[1][2] = (*state)[3][2]; + (*state)[3][2] = temp; + + // Rotate third row 3 columns to left + temp = (*state)[0][3]; + (*state)[0][3] = (*state)[3][3]; + (*state)[3][3] = (*state)[2][3]; + (*state)[2][3] = (*state)[1][3]; + (*state)[1][3] = temp; +} + +static uint8_t xtime(uint8_t x) +{ + return ((x<<1) ^ (((x>>7) & 1) * 0x1b)); +} + +// MixColumns function mixes the columns of the state matrix +static void MixColumns(state_t* state) +{ + uint8_t i; + uint8_t Tmp, Tm, t; + for (i = 0; i < 4; ++i) + { + t = (*state)[i][0]; + Tmp = (*state)[i][0] ^ (*state)[i][1] ^ (*state)[i][2] ^ (*state)[i][3] ; + Tm = (*state)[i][0] ^ (*state)[i][1] ; Tm = xtime(Tm); (*state)[i][0] ^= Tm ^ Tmp ; + Tm = (*state)[i][1] ^ (*state)[i][2] ; Tm = xtime(Tm); (*state)[i][1] ^= Tm ^ Tmp ; + Tm = (*state)[i][2] ^ (*state)[i][3] ; Tm = xtime(Tm); (*state)[i][2] ^= Tm ^ Tmp ; + Tm = (*state)[i][3] ^ t ; Tm = xtime(Tm); (*state)[i][3] ^= Tm ^ Tmp ; + } +} + +uint8_t RoundKey128[176]; // AES_KEYLEN 16 Key length in bytes +uint8_t RoundKey256[240]; // AES_KEYLEN 32 Key length in bytes + +#define KEYNR256 14 +#define KEYNR128 10 + +#define KEYNK256 8 +#define KEYNK128 4 + +#define ENCRYPTNR256 14 +#define ENCRYPTNR128 3 + +// This function produces Nb(Nr+1) round keys. The round keys are used in each round to decrypt the states. +static void KeyExpansion(uint8_t* RoundKey, const uint8_t* Key, int keyNr, int keyNk) +{ + int i, j, k; + uint8_t tempa[4]; // Used for the column/row operations + + // The first round key is the key itself. + for (i = 0; i < keyNk; ++i) + { + RoundKey[(i * 4) + 0] = Key[(i * 4) + 0]; + RoundKey[(i * 4) + 1] = Key[(i * 4) + 1]; + RoundKey[(i * 4) + 2] = Key[(i * 4) + 2]; + RoundKey[(i * 4) + 3] = Key[(i * 4) + 3]; + } + + // All other round keys are found from the previous round keys. + for (i = keyNk; i < Nb * (keyNr + 1); ++i) + { + { + k = (i - 1) * 4; + tempa[0]=RoundKey[k + 0]; + tempa[1]=RoundKey[k + 1]; + tempa[2]=RoundKey[k + 2]; + tempa[3]=RoundKey[k + 3]; + + } + + if (i % keyNk == 0) + { + // This function shifts the 4 bytes in a word to the left once. + // [a0,a1,a2,a3] becomes [a1,a2,a3,a0] + + // Function RotWord() + { + const uint8_t u8tmp = tempa[0]; + tempa[0] = tempa[1]; + tempa[1] = tempa[2]; + tempa[2] = tempa[3]; + tempa[3] = u8tmp; + } + + // SubWord() is a function that takes a four-byte input word and + // applies the S-box to each of the four bytes to produce an output word. + + // Function Subword() + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + + tempa[0] = tempa[0] ^ Rcon[i/keyNk]; + } + + // AES256 only + if ((keyNk==8)&&(i % keyNk == 4)) + { + // Function Subword() + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + } + + j = i * 4; k=(i - keyNk) * 4; + RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0]; + RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1]; + RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2]; + RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3]; + } +} /* * Loads an AES key. Can either be a 16 byte or 32 byte bytearray. */ void aes_load_key(uint8_t *enc_key, int keylen) { - switch (keylen) { - case 16: { - /* 128 bit key setup */ - key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); - key_schedule[1] = KEYEXP128(key_schedule[0], 0x01); - key_schedule[2] = KEYEXP128(key_schedule[1], 0x02); - key_schedule[3] = KEYEXP128(key_schedule[2], 0x04); - key_schedule[4] = KEYEXP128(key_schedule[3], 0x08); - key_schedule[5] = KEYEXP128(key_schedule[4], 0x10); - key_schedule[6] = KEYEXP128(key_schedule[5], 0x20); - key_schedule[7] = KEYEXP128(key_schedule[6], 0x40); - key_schedule[8] = KEYEXP128(key_schedule[7], 0x80); - key_schedule[9] = KEYEXP128(key_schedule[8], 0x1B); - key_schedule[10] = KEYEXP128(key_schedule[9], 0x36); + +#ifndef DISABLE_AESNI + if(!bCheckedAES) + { + uint32_t eax, ebx, ecx, edx; + + eax = ebx = ecx = edx = 0; + __get_cpuid(1, &eax, &ebx, &ecx, &edx); + bHasAES=(ecx & bit_AES) > 0; + bCheckedAES=true; + } + + if(bHasAES) + return ni_aes_load_key(enc_key, keylen); +#endif // DISABLE_AESNI + + switch(keylen){ + case 32: + KeyExpansion(RoundKey256, enc_key, KEYNR256, KEYNK256); break; - } - case 32: { - /* 256 bit key setup */ - key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); - key_schedule[1] = _mm_loadu_si128((const __m128i*) (enc_key+16)); - key_schedule[2] = KEYEXP256(key_schedule[0], key_schedule[1], 0x01); - key_schedule[3] = KEYEXP256_2(key_schedule[1], key_schedule[2]); - key_schedule[4] = KEYEXP256(key_schedule[2], key_schedule[3], 0x02); - key_schedule[5] = KEYEXP256_2(key_schedule[3], key_schedule[4]); - key_schedule[6] = KEYEXP256(key_schedule[4], key_schedule[5], 0x04); - key_schedule[7] = KEYEXP256_2(key_schedule[5], key_schedule[6]); - key_schedule[8] = KEYEXP256(key_schedule[6], key_schedule[7], 0x08); - key_schedule[9] = KEYEXP256_2(key_schedule[7], key_schedule[8]); - key_schedule[10] = KEYEXP256(key_schedule[8], key_schedule[9], 0x10); - key_schedule[11] = KEYEXP256_2(key_schedule[9], key_schedule[10]); - key_schedule[12] = KEYEXP256(key_schedule[10], key_schedule[11], 0x20); - key_schedule[13] = KEYEXP256_2(key_schedule[11], key_schedule[12]); - key_schedule[14] = KEYEXP256(key_schedule[12], key_schedule[13], 0x40); + case 16: + KeyExpansion(RoundKey128, enc_key, KEYNR128, KEYNK128); break; - } } } -// Declares a global variable for efficiency. -__m128i m_global; +/* +* XOR 128 bits +*/ +static inline void xor128(const uint8_t *in1, const uint8_t *in2, uint8_t *out) { + for(int i=0;i<16;i++) { + out[i]=in1[i]^in2[i]; + } +} /* * Encrypts a plaintext using AES256. */ -static inline void aes256_enc(const uint8_t *plainText, uint8_t *cipherText) { - m_global = _mm_loadu_si128(reinterpret_cast(plainText)); +static inline void aes256_enc(const uint8_t *in, uint8_t *out) { + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes256_enc(in, out); +#endif // DISABLE_AESNI + + memcpy(out,in,16); + + state_t *state=(state_t*)out; + + uint8_t round = 0; - DO_ENC_BLOCK_256(m_global, key_schedule); + // Add the First round key to the state before starting the rounds. + AddRoundKey(0, state, RoundKey256); - _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); + // There will be Nr rounds. + // The first Nr-1 rounds are identical. + // These Nr-1 rounds are executed in the loop below. + for (round = 1; round < ENCRYPTNR256; ++round) + { + SubBytes(state); + ShiftRows(state); + MixColumns(state); + AddRoundKey(round, state, RoundKey256); + } + + // The last round is given below. + // The MixColumns function is not here in the last round. + SubBytes(state); + ShiftRows(state); + AddRoundKey(ENCRYPTNR256, state, RoundKey256); } - -/* - * Encrypts a plaintext using AES128 with 2 rounds. - */ -static inline void aes128_enc(const uint8_t *plainText, uint8_t *cipherText) { - m_global = _mm_loadu_si128(reinterpret_cast(plainText)); - - // Uses the 2 round encryption innstead of the full 10 round encryption - DO_ENC_BLOCK_2ROUND(m_global, key_schedule); - - _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); -} - + /* * Encrypts an integer using AES128 with 2 rounds. */ -static inline __m128i aes128_enc_int(__m128i plainText) { - // Uses the 2 round encryption innstead of the full 10 round encryption - DO_ENC_BLOCK_2ROUND(plainText, key_schedule); - return plainText; +static inline void aes128_enc(uint8_t *in, uint8_t *out) { + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_enc(in, out); +#endif // DISABLE_AESNI + + memcpy(out,in,16); + + state_t *state=(state_t*)out; + + uint8_t round = 0; + + // Add the First round key to the state before starting the rounds. + AddRoundKey(0, state, RoundKey128); + + // There will be Nr rounds. + // The first Nr-1 rounds are identical. + // These Nr-1 rounds are executed in the loop below. + for (round = 1; round < ENCRYPTNR128; ++round) + { + SubBytes(state); + ShiftRows(state); + MixColumns(state); + AddRoundKey(round, state, RoundKey128); + } } - -__m128i m1; -__m128i m2; -__m128i m3; -__m128i m4; - + /* * Uses AES cache mode to map a 2 block ciphertext into 128 bit result. */ static inline void aes128_2b(uint8_t *block1, uint8_t *block2, uint8_t *res) { - m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); - m3 = aes128_enc_int(m1); // E(L) - m3 = aes128_enc_int(_mm_xor_si128(m3, m2)); - _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); -} + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_2b(block1, block2, res); +#endif // DISABLE_AESNI + + uint8_t m1[16]; + uint8_t m2[16]; + uint8_t m3[16]; + uint8_t intermediate[16]; + memcpy(m1,block1,16); + memcpy(m2,block2,16); + + aes128_enc(m1,m3); + xor128(m3, m2, intermediate); + aes128_enc(intermediate,m3); + + memcpy(res,m3,16); +} + /* * Uses AES cache mode to map a 3 block ciphertext into 128 bit result. */ static inline void aes128_3b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* res) { - m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_3b(block1, block2, block3, res); +#endif // DISABLE_AESNI + + uint8_t m1[16]; + uint8_t m2[16]; + uint8_t m3[16]; - m1 = aes128_enc_int(m1); // E(La) - m2 = aes128_enc_int(m2); // E(Ra) + memcpy(m1,block1,16); + memcpy(m2,block2,16); - m1 = _mm_xor_si128(m1, m2); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); - - m2 = aes128_enc_int(m2); - m1 = _mm_xor_si128(m1, m2); - m3 = aes128_enc_int(m1); - _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); + aes128_enc(m1,m1); // E(La) + aes128_enc(m2,m2); // E(Ra) + + xor128(m1, m2, m1); + memcpy(m2,block3,16); + + aes128_enc(m2,m2); + xor128(m1, m2, m1); + aes128_enc(m1,m3); + memcpy(res,m3,16); } /* * Uses AES cache mode to map a 4 block ciphertext into 128 bit result. */ static inline void aes128_4b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* block4, uint8_t* res) { - m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); - m3 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); - m4 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block4)); + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_4b(block1, block2, block3, block4, res); +#endif // DISABLE_AESNI - m1 = aes128_enc_int(m1); // E(La) - m1 = _mm_xor_si128(m1, m3); - m1 = aes128_enc_int(m1); // E(E(La) ^ Lb) - m2 = aes128_enc_int(m2); // E(Ra) + uint8_t m1[16]; + uint8_t m2[16]; + uint8_t m3[16]; + uint8_t m4[16]; - m1 = _mm_xor_si128(m1, m2); // xor e(Ra) - m1 = _mm_xor_si128(m1, m4); // xor Rb + memcpy(m1,block1,16); + memcpy(m2,block2,16); + memcpy(m3,block3,16); + memcpy(m4,block4,16); - m3 = aes128_enc_int(m1); - _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); + aes128_enc(m1,m1); // E(La) + xor128(m1, m3, m1); + aes128_enc(m1,m1); // E(E(La) ^ Lb) + aes128_enc(m2,m2); // E(Ra) + + xor128(m1, m2, m1); // xor e(Ra) + xor128(m1, m4, m1); // xor Rb + + aes128_enc(m1,m3); // E(La) + memcpy(res,m3,16); } -#endif // SRC_CPP_AES_HPP_ diff --git a/lib/chiapos/src/aesni.hpp b/lib/chiapos/src/aesni.hpp new file mode 100644 index 00000000..d3d469d5 --- /dev/null +++ b/lib/chiapos/src/aesni.hpp @@ -0,0 +1,270 @@ +// Copyright 2018 Chia Network Inc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Some public domain code is taken from pycrypto: +// https://github.com/dlitz/pycrypto/blob/master/src/AESNI.c +// +// AESNI.c: AES using AES-NI instructions +// +// Written in 2013 by Sebastian Ramacher + +#ifndef SRC_CPP_AES_HPP_ +#define SRC_CPP_AES_HPP_ + +#include // for memcmp +#include // for intrinsics for AES-NI + +/** + * Encrypts a message of 128 bits with a 128 bit key, using + * 10 rounds of AES128 (9 full rounds and one final round). Uses AES-NI + * assembly instructions. + */ +#define DO_ENC_BLOCK_128(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[0]); \ + m = _mm_aesenc_si128(m, k[1]); \ + m = _mm_aesenc_si128(m, k[2]); \ + m = _mm_aesenc_si128(m, k[3]); \ + m = _mm_aesenc_si128(m, k[4]); \ + m = _mm_aesenc_si128(m, k[5]); \ + m = _mm_aesenc_si128(m, k[6]); \ + m = _mm_aesenc_si128(m, k[7]); \ + m = _mm_aesenc_si128(m, k[8]); \ + m = _mm_aesenc_si128(m, k[9]); \ + m = _mm_aesenclast_si128(m, k[10]); \ + } while (0) + +/** + * Encrypts a message of 128 bits with a 256 bit key, using + * 13 rounds of AES256 (13 full rounds and one final round). Uses + * AES-NI assembly instructions. + */ +#define DO_ENC_BLOCK_256(m, k) \ + do {\ + m = _mm_xor_si128(m, k[ 0]); \ + m = _mm_aesenc_si128(m, k[ 1]); \ + m = _mm_aesenc_si128(m, k[ 2]); \ + m = _mm_aesenc_si128(m, k[ 3]); \ + m = _mm_aesenc_si128(m, k[ 4]); \ + m = _mm_aesenc_si128(m, k[ 5]); \ + m = _mm_aesenc_si128(m, k[ 6]); \ + m = _mm_aesenc_si128(m, k[ 7]); \ + m = _mm_aesenc_si128(m, k[ 8]); \ + m = _mm_aesenc_si128(m, k[ 9]); \ + m = _mm_aesenc_si128(m, k[ 10]);\ + m = _mm_aesenc_si128(m, k[ 11]);\ + m = _mm_aesenc_si128(m, k[ 12]);\ + m = _mm_aesenc_si128(m, k[ 13]);\ + m = _mm_aesenclast_si128(m, k[ 14]);\ + }while(0) + +/** + * Encrypts a message of 128 bits with a 128 bit key, using + * 2 full rounds of AES128. Uses AES-NI assembly instructions. + */ +#define DO_ENC_BLOCK_2ROUND(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[0]); \ + m = _mm_aesenc_si128(m, k[1]); \ + m = _mm_aesenc_si128(m, k[2]); \ + } while (0) +/** + * Decrypts a ciphertext of 128 bits with a 128 bit key, using + * 10 rounds of AES128 (9 full rounds and one final round). + * Uses AES-NI assembly instructions. + */ +#define DO_DEC_BLOCK(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[10 + 0]); \ + m = _mm_aesdec_si128(m, k[10 + 1]); \ + m = _mm_aesdec_si128(m, k[10 + 2]); \ + m = _mm_aesdec_si128(m, k[10 + 3]); \ + m = _mm_aesdec_si128(m, k[10 + 4]); \ + m = _mm_aesdec_si128(m, k[10 + 5]); \ + m = _mm_aesdec_si128(m, k[10 + 6]); \ + m = _mm_aesdec_si128(m, k[10 + 7]); \ + m = _mm_aesdec_si128(m, k[10 + 8]); \ + m = _mm_aesdec_si128(m, k[10 + 9]); \ + m = _mm_aesdeclast_si128(m, k[0]); \ + } while (0) + +/** + * Decrypts a ciphertext of 128 bits with a 128 bit key, using + * 2 full rounds of AES128. Uses AES-NI assembly instructions. + * Will not work unless key schedule is modified. + */ +/* +#define DO_DEC_BLOCK_2ROUND(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[2 + 0]); \ + m = _mm_aesdec_si128(m, k[2 + 1]); \ + m = _mm_aesdec_si128(m, k[2 + 2]); \ + } while (0) +*/ + +static __m128i key_schedule[20]; // The expanded key + +static __m128i aes128_keyexpand(__m128i key) { + key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); + key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); + return _mm_xor_si128(key, _mm_slli_si128(key, 4)); +} + +#define KEYEXP128_H(K1, K2, I, S) _mm_xor_si128(aes128_keyexpand(K1), \ + _mm_shuffle_epi32(_mm_aeskeygenassist_si128(K2, I), S)) + +#define KEYEXP128(K, I) KEYEXP128_H(K, K, I, 0xff) +#define KEYEXP256(K1, K2, I) KEYEXP128_H(K1, K2, I, 0xff) +#define KEYEXP256_2(K1, K2) KEYEXP128_H(K1, K2, 0x00, 0xaa) + +// public API + +/* + * Loads an AES key. Can either be a 16 byte or 32 byte bytearray. + */ +void ni_aes_load_key(uint8_t *enc_key, int keylen) { + switch (keylen) { + case 16: { + /* 128 bit key setup */ + key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); + key_schedule[1] = KEYEXP128(key_schedule[0], 0x01); + key_schedule[2] = KEYEXP128(key_schedule[1], 0x02); + key_schedule[3] = KEYEXP128(key_schedule[2], 0x04); + key_schedule[4] = KEYEXP128(key_schedule[3], 0x08); + key_schedule[5] = KEYEXP128(key_schedule[4], 0x10); + key_schedule[6] = KEYEXP128(key_schedule[5], 0x20); + key_schedule[7] = KEYEXP128(key_schedule[6], 0x40); + key_schedule[8] = KEYEXP128(key_schedule[7], 0x80); + key_schedule[9] = KEYEXP128(key_schedule[8], 0x1B); + key_schedule[10] = KEYEXP128(key_schedule[9], 0x36); + break; + } + case 32: { + /* 256 bit key setup */ + key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); + key_schedule[1] = _mm_loadu_si128((const __m128i*) (enc_key+16)); + key_schedule[2] = KEYEXP256(key_schedule[0], key_schedule[1], 0x01); + key_schedule[3] = KEYEXP256_2(key_schedule[1], key_schedule[2]); + key_schedule[4] = KEYEXP256(key_schedule[2], key_schedule[3], 0x02); + key_schedule[5] = KEYEXP256_2(key_schedule[3], key_schedule[4]); + key_schedule[6] = KEYEXP256(key_schedule[4], key_schedule[5], 0x04); + key_schedule[7] = KEYEXP256_2(key_schedule[5], key_schedule[6]); + key_schedule[8] = KEYEXP256(key_schedule[6], key_schedule[7], 0x08); + key_schedule[9] = KEYEXP256_2(key_schedule[7], key_schedule[8]); + key_schedule[10] = KEYEXP256(key_schedule[8], key_schedule[9], 0x10); + key_schedule[11] = KEYEXP256_2(key_schedule[9], key_schedule[10]); + key_schedule[12] = KEYEXP256(key_schedule[10], key_schedule[11], 0x20); + key_schedule[13] = KEYEXP256_2(key_schedule[11], key_schedule[12]); + key_schedule[14] = KEYEXP256(key_schedule[12], key_schedule[13], 0x40); + break; + } + } +} + +// Declares a global variable for efficiency. +__m128i m_global; + +/* + * Encrypts a plaintext using AES256. + */ +static inline void ni_aes256_enc(const uint8_t *plainText, uint8_t *cipherText) { + m_global = _mm_loadu_si128(reinterpret_cast(plainText)); + + DO_ENC_BLOCK_256(m_global, key_schedule); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); +} + +/* + * Encrypts a plaintext using AES128 with 2 rounds. + */ +static inline void ni_aes128_enc(const uint8_t *plainText, uint8_t *cipherText) { + m_global = _mm_loadu_si128(reinterpret_cast(plainText)); + + // Uses the 2 round encryption innstead of the full 10 round encryption + DO_ENC_BLOCK_2ROUND(m_global, key_schedule); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); +} + +/* + * Encrypts an integer using AES128 with 2 rounds. + */ +static inline __m128i ni_aes128_enc_int(__m128i plainText) { + // Uses the 2 round encryption innstead of the full 10 round encryption + DO_ENC_BLOCK_2ROUND(plainText, key_schedule); + return plainText; +} + +__m128i m1; +__m128i m2; +__m128i m3; +__m128i m4; + +/* + * Uses AES cache mode to map a 2 block ciphertext into 128 bit result. + */ +static inline void ni_aes128_2b(uint8_t *block1, uint8_t *block2, uint8_t *res) { + m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + m3 = ni_aes128_enc_int(m1); // E(L) + m3 = ni_aes128_enc_int(_mm_xor_si128(m3, m2)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); +} + +/* + * Uses AES cache mode to map a 3 block ciphertext into 128 bit result. + */ +static inline void ni_aes128_3b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* res) { + m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + + m1 = ni_aes128_enc_int(m1); // E(La) + m2 = ni_aes128_enc_int(m2); // E(Ra) + + m1 = _mm_xor_si128(m1, m2); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); + + m2 = ni_aes128_enc_int(m2); + m1 = _mm_xor_si128(m1, m2); + m3 = ni_aes128_enc_int(m1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); +} + +/* + * Uses AES cache mode to map a 4 block ciphertext into 128 bit result. + */ +static inline void ni_aes128_4b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* block4, uint8_t* res) { + m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); + m3 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + m4 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block4)); + + m1 = ni_aes128_enc_int(m1); // E(La) + m1 = _mm_xor_si128(m1, m3); + m1 = ni_aes128_enc_int(m1); // E(E(La) ^ Lb) + m2 = ni_aes128_enc_int(m2); // E(Ra) + + m1 = _mm_xor_si128(m1, m2); // xor e(Ra) + m1 = _mm_xor_si128(m1, m4); // xor Rb + + m3 = ni_aes128_enc_int(m1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); +} + +#endif // SRC_CPP_AES_HPP_ diff --git a/lib/chiapos/src/cmake_aesni_test.cpp b/lib/chiapos/src/cmake_aesni_test.cpp new file mode 100644 index 00000000..daed6189 --- /dev/null +++ b/lib/chiapos/src/cmake_aesni_test.cpp @@ -0,0 +1,16 @@ +#include "aes.hpp" + +int main() { + uint8_t enc_key[32]; + uint8_t in[16]; + uint8_t out[16]; + + memset(enc_key,0x00,sizeof(enc_key)); + memset(in,0x00,sizeof(in)); + + aes_load_key(enc_key, sizeof(enc_key)); + aes256_enc(in, out); + + return 0; +} + diff --git a/requirements.txt b/requirements.txt index d0803f56..7619a7cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,8 @@ yarl==1.4.2 zipp==2.0.0 sortedcontainers==2.1.0 -e lib/chiapos +-e lib/bip158 -e lib/py-setproctitle -e lib/chiavdf/fast_vdf --e git+https://github.com/Chia-Network/clvm.git@134c2f92d575a542272ce0cbe59c167b255e50ae#egg=clvm --e git+https://github.com/Chia-Network/clvm_tools.git@208dc0bbf41e16d8e5dd8cbc1510e1528a48bbe1#egg=clvm_tools +-e git+https://github.com/Chia-Network/clvm.git@bb538804062c01f999a228c4fc1e17a6e2835851#egg=clvm +-e git+https://github.com/Chia-Network/clvm_tools.git@343a82f16f3da1d042283f9f8969315e0c71bcf5#egg=clvm_tools diff --git a/src/consensus/constants.py b/src/consensus/constants.py index 948d2f96..42e077ee 100644 --- a/src/consensus/constants.py +++ b/src/consensus/constants.py @@ -24,8 +24,8 @@ constants: Dict[str, Any] = { # block, to allow better leaders to be released first. This is a slow block. "PROPAGATION_DELAY_THRESHOLD": 1500, # Hardcoded genesis block, generated using tests/block_tools.py - # Replace this any time the above constants change. - "GENESIS_BLOCK": b'\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x15N3\xd3\xf9H\xc2K\x96\xfe\xf2f\xa2\xbf\x87\x0e\x0f,\xd0\xd4\x0f6s\xb1".\\\xf5\x8a\xb4\x03\x84\x8e\xf9\xbb\xa1\xca\xdef3:\xe4?\x0c\xe5\xc6\x12\x80\x88\xbe_6 X\xf1\x83\xe8\x99\xdf)\xb8\xf6t\xe0;\x82\x17\xc5\xe5\x94\xb7\xef\xc2|\x94\xe6\xfb\x91L\x85\xe4\x00WVV\xefJ\x1e/>\xf6\xc5Gr5n\x13\x00\x00\x00\x98\xe4\xd8(mep\xcf}\xdb\xd7(\x04N"\xd1I\x18g\xae[\xff\xc0#z\xee\xb7\xbd3f\xe4zR3mi-\x89\x88\xbc\xd3\xf0|\xee\x03\x13\xc9}\xbb\x9b\x7f\x7f\xcfj\x08\x01\xe0*\x1e\x9an\xf6\xba\xd5\xb1\xc1\x80\x96\x8a\x99\xe3\x91\x92j\xce\xfdij\xea\xccT\xd0[\xd0\x89\xdc\xb8\xa3 /\xf27\x0f\x9ce\x87\x9dK\xe7\xab\x01\xbb\x1e\x91U\x95\x0f\xc0c\xa3\xa4\x81Um\x80_\xee\x8f3\xc7\xe3?\xf5\xacyF\x941\x90\x9e\xd1\xd0\x0bB\xa4\xa4\xe18\x13\xd5x\xca\xbd\x9b;\xf9B\xa1y\xaasm\x14\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x03\xf9\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^\x1d\x06\xb8_\x85B=\xa2q\x01\xd6\x149\x18\xd1^\xa6F\'jB*&\x8d\xfa\xdf\x82q\xb5m\xb6\xdc)S\x87kiS\xa1D\xbd9\xb7bcro\x00\xad\xc0=\xdd\xd9\x08\xb1\xdcV\x03Vj\x101\xef\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf6\xc9\xf6\xe2\xe3\xea\xfet)\xd2K\x7f-\x19\x0ew\x16\xd7\xac\xa4;\xd9\xde\xc89\x8c\xe0E\xe7\x151\x85\xca\x03n/\xd0]Z\xb7F*zQ\xb6e\x8a\xc2\xe8\xb7\xb8N.\xea\xc4\xdd\x7f\x83\xb6a\xa3\x0b\xf0\xad\x03\x00\x00\x03\xa6\x00\x0e\t\x80[\xf0Kp&\x9e\xfd\x94y\xbf.\x935\x9fE&"b\xdb\tqj\xf8\x85S\xe1\xef=8\xb4\xf0\xa4_6\x88\x85\xf5\xe0\x831(\x9b"\xb01\xba\xe0\xfc%+\xbf.\xa6\x1e\x7f\xf4{\xe9HN\x03\xff\xf9\x96\xc8\x84y)\t\xba\xe4\x1c\xe8\xf6\xd4g\x9f\xb3\x11\x1a,\xa53\x07E\x0c\x11\x9c\xfd\x90\xa4\x1a\xf7D\x8e\x99\xca\xcb*\x9bu\x7f8\x03\xe3m\xca\xc5\x83{v\x81\xffH\x8d\xd7\xcf\xf5(q\xd8\x1a;\xe8^\x17\x00\x00\x00\x00\x00\x00\x15\xce\x00C<\xe4\xa6J\xea\x9d\x90\x15cMK\xce+\xec\x88\xd3\xe9\xa0!\x81\x91pS\xa0\xe8\x13\xab\xcf\xcc;\xc2L=[\'\xaf\x0c\xc7\\\xd1\\\xfc\xba\xb5Xoz\\\xdfW\xf6Y\xa46\x0cwep^%u\x996^\xff\xe5:\x1dc\x8c\x10\x19\xd0\xb6\xb9$\\$n\xfe?\x06diZ\xb2\xb4\x98\xb4\xb2\x8c\xc1\xfdvp\xd0!l\x90M\xdf\x19\x04\xe5\x8c\x04\xb4\xe97\xce;\xdd\xcb\x00A`\xfax\xce\xdf\x92=K\xb1G8\x9c\xdc\x89\x00\x00\x00\x00\x00\x03h,\x009D\x93\xbf\x06\xc0\x8b\\\xfd\xf2\xafb\xc7o%\x10\xd2c\'\xa1\xaal\xc7k\xc9\xbc/0\xb7\x88\xa5z\xcd\xf5G\\\x846\xb1\xde\x91\xf3/d\x9c\xfc\xfa\xaf\x1c0e\x9f\xe6\x94\xba\xfa\xc2\xccg^\xb1\x86\x0c\x8c\xff\xe5j\x19\xd5\xcdfe\xabP\xbf\xef\'\xd5\xd9\xbe\x95\xf6\xcc\xf2I\xed\xa8@\x0eWV\x05tN\xf6?\xd5G_\xfa\x17\xd9\xb1\xf1n\xcd\xc8\x86\xb1\x9c\xc5\xf1;Z\xdb6\xd8p\xb1\x17\xca=\x8c\xaf9\x88%\xea\x19\x00$I09#S\x16m\xe5\xc2\xb3;w\xe9\xd2+)K\xac\xc6\x83\x8f\x15|\xcb\xf66V\x9a \xa1$\xce-\x0c\x83\'\x01a\x86&\xad_\xe7\xdbd\xb0\xbf\x1d\x15\x82e}\xd8+\x819\xdf\x9f\xfa\xc9\xbd\xaeE\x00\x15\xc1qa\xd5\xa3\xe4\xaf\x0f\x9b\x15O\x0fG\x1eF\xbf\xc5\x96c\xca\xd3_\xaa\xf3\xdet;\r\xd5S\xd2\xfd\xdf:\xcc\x0c\xfd\x9bT\xce1\xa3xl;^Zf\xf4Qs\xc0|1\x16\xbc\n\xdd\x90\xf6\xa7y\xf3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^U\xefb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00>\x93h\xb0E\x0c\xaa\xda1\x9a\x04\x83 \xedGe\xf1\'\xab\xc7Z\x9d\xaf!\x18D#\t\x0bz\xe2\xc4\x0cN\x8b\x9f\xf7\x1fs\x80\x91\x15\x9f\xce\xfcv\xdf\x173&q\x8c)\x9b\x16u\x15\xb2x\xf4\xd9r~\xf0\x00\x00\x00\x00\x00\x00\x01\xf4\x00\x00\x00\x00\x00\x03\xf9\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\xdc\xf2\xc9\x96\xabXM\x15s \xa0O\xa7\xf4\xde\xe5\x9a\x80\x7fv\xaf\x19\xc0\xa2\x1d\x83kqCH\x82YA\x194\x81N\x9do-s\x82c"\xca[N\x06)\x0e\x8d0\x95\x0b\xed\x7fu\x16\x1a-\xf7`B\xca\rn\xfb\x06(\x81\xd6A\x8f\x1a\xab\x0e\x04\x9a\xff*m\xecHIo\\\x0f&"x\x86R\xf8\x8b\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8b)\xaa\x96x8\xd76J\xa6\x8b[\x98\t\xe0\\\xe3^7qD\x8c\xf5q\x08\xf2\xa2\xc9\xb03mv\x00\x00\x0c\xbb\xa1\x06\xe0\x00N\x1f\xe8;}6F\xd7\xec\xc7\x83\x16T\x96\x1f\xe6\x88,\xa4\x9b\xa3Lo\xd0\xe6\x89jW\xac\xba\xae)\xe9\x91?\x97\x0fU\xf5\xd8\xdc\x9e\xce\xbf~\xad\xc2\xbc\x17v|\x947N\x0e\xfa\xff\xe6;\xce@|\xe9{\xe2:\xa8H\xb4\xb9\xde;<;-\x9a\x03\xbf\xa3\xff\xed\x81\x0cd\x80|(I\x9e\x8c\xa5\x83\xdf\x8a\x1aX\xc1#\x19uE`)\xeblV\x1d\x8f\xe6\x1f\xfa\x03\xe2\xf4b\xdfO\x9c\x11\x1fHJ2\xbfvC\x8b^\x8b)\xaa\x96x8\xd76J\xa6\x8b[\x98\t\xe0\\\xe3^7qD\x8c\xf5q\x08\xf2\xa2\xc9\xb03mv\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', # noqa: E501, + # Replace this any time the constants change. + "GENESIS_BLOCK": b"\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x15N3\xd3\xf9H\xc2K\x96\xfe\xf2f\xa2\xbf\x87\x0e\x0f,\xd0\xd4\x0f6s\xb1\".\\\xf5\x8a\xb4\x03\x84\x8e\xf9\xbb\xa1\xca\xdef3:\xe4?\x0c\xe5\xc6\x12\x80\x88\xbe_6 X\xf1\x83\xe8\x99\xdf)\xb8\xf6t\xe0;\x82\x17\xc5\xe5\x94\xb7\xef\xc2|\x94\xe6\xfb\x91L\x85\xe4\x00WVV\xefJ\x1e/>\xf6\xc5Gr5n\x13\x00\x00\x00\x98\xe4\xd8(mep\xcf}\xdb\xd7(\x04N\"\xd1I\x18g\xae[\xff\xc0#z\xee\xb7\xbd3f\xe4zR3mi-\x89\x88\xbc\xd3\xf0|\xee\x03\x13\xc9}\xbb\x9b\x7f\x7f\xcfj\x08\x01\xe0*\x1e\x9an\xf6\xba\xd5\xb1\xc1\x80\x96\x8a\x99\xe3\x91\x92j\xce\xfdij\xea\xccT\xd0[\xd0\x89\xdc\xb8\xa3 /\xf27\x0f\x9ce\x87\x9dK\xe7\xab\x01\xbb\x1e\x91U\x95\x0f\xc0c\xa3\xa4\x81Um\x80_\xee\x8f3\xc7\xe3?\xf5\xacyF\x941\x90\x9e\xd1\xd0\x0bB\xa4\xa4\xe18\x13\xd5x\xca\xbd\x9b;\xf9B\xa1y\xaasm\x14\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x03\xf9\x89\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^\x1d\x06\xb8_\x85B=\xa2q\x01\xd6\x149\x18\xd1^\xa6F'jB*&\x8d\xfa\xdf\x82q\xb5m\xb6\xdc)S\x87kiS\xa1D\xbd9\xb7bcro\x00\xad\xc0=\xdd\xd9\x08\xb1\xdcV\x03Vj\x101\xef\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf6\xc9\xf6\xe2\xe3\xea\xfet)\xd2K\x7f-\x19\x0ew\x16\xd7\xac\xa4;\xd9\xde\xc89\x8c\xe0E\xe7\x151\x85\xca\x03n/\xd0]Z\xb7F*zQ\xb6e\x8a\xc2\xe8\xb7\xb8N.\xea\xc4\xdd\x7f\x83\xb6a\xa3\x0b\xf0\xad\x03\x00\x00\x03\xa6\x00\x0e\t\x80[\xf0Kp&\x9e\xfd\x94y\xbf.\x935\x9fE&\"b\xdb\tqj\xf8\x85S\xe1\xef=8\xb4\xf0\xa4_6\x88\x85\xf5\xe0\x831(\x9b\"\xb01\xba\xe0\xfc%+\xbf.\xa6\x1e\x7f\xf4{\xe9HN\x03\xff\xf9\x96\xc8\x84y)\t\xba\xe4\x1c\xe8\xf6\xd4g\x9f\xb3\x11\x1a,\xa53\x07E\x0c\x11\x9c\xfd\x90\xa4\x1a\xf7D\x8e\x99\xca\xcb*\x9bu\x7f8\x03\xe3m\xca\xc5\x83{v\x81\xffH\x8d\xd7\xcf\xf5(q\xd8\x1a;\xe8^\x17\x00\x00\x00\x00\x00\x00\x15\xce\x00C<\xe4\xa6J\xea\x9d\x90\x15cMK\xce+\xec\x88\xd3\xe9\xa0!\x81\x91pS\xa0\xe8\x13\xab\xcf\xcc;\xc2L=['\xaf\x0c\xc7\\\xd1\\\xfc\xba\xb5Xoz\\\xdfW\xf6Y\xa46\x0cwep^%u\x996^\xff\xe5:\x1dc\x8c\x10\x19\xd0\xb6\xb9$\\$n\xfe?\x06diZ\xb2\xb4\x98\xb4\xb2\x8c\xc1\xfdvp\xd0!l\x90M\xdf\x19\x04\xe5\x8c\x04\xb4\xe97\xce;\xdd\xcb\x00A`\xfax\xce\xdf\x92=K\xb1G8\x9c\xdc\x89\x00\x00\x00\x00\x00\x03h,\x009D\x93\xbf\x06\xc0\x8b\\\xfd\xf2\xafb\xc7o%\x10\xd2c'\xa1\xaal\xc7k\xc9\xbc/0\xb7\x88\xa5z\xcd\xf5G\\\x846\xb1\xde\x91\xf3/d\x9c\xfc\xfa\xaf\x1c0e\x9f\xe6\x94\xba\xfa\xc2\xccg^\xb1\x86\x0c\x8c\xff\xe5j\x19\xd5\xcdfe\xabP\xbf\xef'\xd5\xd9\xbe\x95\xf6\xcc\xf2I\xed\xa8@\x0eWV\x05tN\xf6?\xd5G_\xfa\x17\xd9\xb1\xf1n\xcd\xc8\x86\xb1\x9c\xc5\xf1;Z\xdb6\xd8p\xb1\x17\xca=\x8c\xaf9\x88%\xea\x19\x00$I09#S\x16m\xe5\xc2\xb3;w\xe9\xd2+)K\xac\xc6\x83\x8f\x15|\xcb\xf66V\x9a \xa1$\xce-\x0c\x83'\x01a\x86&\xad_\xe7\xdbd\xb0\xbf\x1d\x15\x82e}\xd8+\x819\xdf\x9f\xfa\xc9\xbd\xaeE\x00\x15\xc1qa\xd5\xa3\xe4\xaf\x0f\x9b\x15O\x0fG\x1eF\xbf\xc5\x96c\xca\xd3_\xaa\xf3\xdet;\r\xd5S\xd2\xfd\xdf:\xcc\x0c\xfd\x9bT\xce1\xa3xl;^Zf\xf4Qs\xc0|1\x16\xbc\n\xdd\x90\xf6\xa7y\xf3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^V\x19\xeb\x00\x00\x00\x04\x01\x1f\xa6H>\x93h\xb0E\x0c\xaa\xda1\x9a\x04\x83 \xedGe\xf1'\xab\xc7Z\x9d\xaf!\x18D#\t\x0bz\xe2\xc4\x0cN\x8b\x9f\xf7\x1fs\x80\x91\x15\x9f\xce\xfcv\xdf\x173&q\x8c)\x9b\x16u\x15\xb2x\xf4\xd9r~\xf0\x00\x00\x00\x00\x00\x00\x01\xf4\x00\x00\x00\x00\x00\x03\xf9\x89\x8d\xbbft\x7f\xc0\xd1\x8b\xc6\x8b\x90}\xb0zM\x89\xcd)\x87\x96=\xdb1Kw\xb6;\xf7\xd0\xee\xdd\xef\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Uo\x84\xc5`c\xd2o)\xbe\xfb+:\x82>\x10]\x0bz\x9b\xee\xf7S\tS|\xbd\xc6\xf9\xa6\xa0K\x0e\xffIIP\xaa\xe8fV\x8e\x14\xe9\xff\xc8\x02\xd3\x13\xeb\x1a\xe9\xf5X\x97y\xd2\x89mA\x05[\x9c%\r\x18W\xc8\x07N\xc3\xbe\x14\xad\xf6c\xf4\xc5\x92\xbe\x00-\x03\xf6d\xb9Nb\xc1\xf1\xf7\n\x15A\x92\xb2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8b)\xaa\x96x8\xd76J\xa6\x8b[\x98\t\xe0\\\xe3^7qD\x8c\xf5q\x08\xf2\xa2\xc9\xb03mv\x00\x00\x0c\xbb\xa1\x06\xe0\x00N\x1f\xe8;}6F\xd7\xec\xc7\x83\x16T\x96\x1f\xe6\x88,\xa4\x9b\xa3Lo\xd0\xe6\x89jW\xac\xba\xae)\xe9\x91?\x97\x0fU\xf5\xd8\xdc\x9e\xce\xbf~\xad\xc2\xbc\x17v|\x947N\x0e\xfa\xff\xe6;\xce@|\xe9{\xe2:\xa8H\xb4\xb9\xde;<;-\x9a\x03\xbf\xa3\xff\xed\x81\x0cd\x80|(I\x9e\x8c\xa5\x83\xdf\x8a\x1aX\xc1#\x19uE`)\xeblV\x1d\x8f\xe6\x1f\xfa\x03\xe2\xf4b\xdfO\x9c\x11\x1fHJ2\xbfvC\x8b^\x8b)\xaa\x96x8\xd76J\xa6\x8b[\x98\t\xe0\\\xe3^7qD\x8c\xf5q\x08\xf2\xa2\xc9\xb03mv\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", # noqa: E501 # Target tx count per sec "TX_PER_SEC": 20, # Size of mempool = 10x the size of block diff --git a/src/farmer.py b/src/farmer.py index 112f1b2f..8cfa0bd4 100644 --- a/src/farmer.py +++ b/src/farmer.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, Set from blspy import PrivateKey, Util from src.consensus.block_rewards import calculate_block_reward -from src.consensus.constants import constants +from src.consensus.constants import constants as consensus_constants from src.consensus.pot_iterations import calculate_iterations_quality from src.pool import create_coinbase_coin_and_signature from src.protocols import farmer_protocol, harvester_protocol @@ -23,7 +23,7 @@ HARVESTER PROTOCOL (FARMER <-> HARVESTER) class Farmer: - def __init__(self, farmer_config: Dict, key_config: Dict): + def __init__(self, farmer_config: Dict, key_config: Dict, override_constants={}): self.config = farmer_config self.key_config = key_config self.harvester_responses_header_hash: Dict[bytes32, bytes32] = {} @@ -39,6 +39,9 @@ class Farmer: self.current_weight: uint64 = uint64(0) self.coinbase_rewards: Dict[uint32, Any] = {} self.proof_of_time_estimate_ips: uint64 = uint64(10000) + self.constants = consensus_constants.copy() + for key, value in override_constants.items(): + self.constants[key] = value async def _on_connect(self): # Sends a handshake to the harvester @@ -81,7 +84,7 @@ class Farmer: challenge_response.plot_size, difficulty, self.proof_of_time_estimate_ips, - constants["MIN_BLOCK_TIME"], + self.constants["MIN_BLOCK_TIME"], ) if height < 1000: # As the difficulty adjusts, don't fetch all qualities if challenge_response.challenge_hash not in self.challenge_to_best_iters: @@ -160,7 +163,7 @@ class Farmer: response.proof.size, difficulty, self.proof_of_time_estimate_ips, - constants["MIN_BLOCK_TIME"], + self.constants["MIN_BLOCK_TIME"], ) estimate_secs: float = number_iters / self.proof_of_time_estimate_ips diff --git a/tests/test_transactions.py b/src/full_node/__init__.py similarity index 100% rename from tests/test_transactions.py rename to src/full_node/__init__.py diff --git a/src/blockchain.py b/src/full_node/blockchain.py similarity index 98% rename from src/blockchain.py rename to src/full_node/blockchain.py index 5f80879e..61d9f8ea 100644 --- a/src/blockchain.py +++ b/src/full_node/blockchain.py @@ -14,15 +14,15 @@ from src.consensus.pot_iterations import ( calculate_ips_from_iterations, calculate_iterations_quality, ) -from src.store import FullNodeStore +from src.full_node.store import FullNodeStore from src.types.full_block import FullBlock, additions_for_npc -from src.types.hashable.Coin import Coin -from src.types.hashable.CoinRecord import CoinRecord +from src.types.hashable.coin import Coin +from src.types.hashable.coin_record import CoinRecord from src.types.header_block import HeaderBlock from src.types.header import Header from src.types.sized_bytes import bytes32 -from src.coin_store import CoinStore +from src.full_node.coin_store import CoinStore from src.util.ConsensusError import Err from src.util.blockchain_check_conditions import blockchain_check_conditions_dict from src.util.condition_tools import hash_key_pairs_for_conditions_dict @@ -82,7 +82,7 @@ class Blockchain: in the consensus constants config. """ self = Blockchain() - self.constants = consensus_constants + self.constants = consensus_constants.copy() for key, value in override_constants.items(): self.constants[key] = value self.tips = [] @@ -744,8 +744,7 @@ class Blockchain: pool.shutdown(wait=True) return results - @staticmethod - def pre_validate_block_multi(data) -> Tuple[bool, Optional[bytes]]: + def pre_validate_block_multi(self, data) -> Tuple[bool, Optional[bytes]]: """ Validates all parts of FullBlock that don't need to be serially checked """ @@ -755,9 +754,7 @@ class Blockchain: return False, None # 4. Check PoT - if not block.proof_of_time.is_valid( - consensus_constants["DISCRIMINANT_SIZE_BITS"] - ): + if not block.proof_of_time.is_valid(self.constants["DISCRIMINANT_SIZE_BITS"]): return False, None # 9. Check harvester signature of header data is valid based on harvester key @@ -927,9 +924,7 @@ class Blockchain: if not block.body.transactions: return Err.UNKNOWN # Get List of names removed, puzzles hashes for removed coins and conditions crated - error, npc_list, cost = await get_name_puzzle_conditions( - block.body.transactions - ) + error, npc_list, cost = get_name_puzzle_conditions(block.body.transactions) if cost > 6000: return Err.BLOCK_COST_EXCEEDS_MAX @@ -953,7 +948,7 @@ class Blockchain: # Check additions for max coin amount for coin in additions: additions_dic[coin.name()] = coin - if coin.amount >= consensus_constants["MAX_COIN_AMOUNT"]: + if coin.amount >= self.constants["MAX_COIN_AMOUNT"]: return Err.COIN_AMOUNT_EXCEEDS_MAXIMUM # Watch out for duplicate outputs @@ -1008,7 +1003,6 @@ class Blockchain: # Check coinbase reward if fees + fee_base != block.body.fees_coin.amount: - print("Fees,", fees, fee_base, block.body.fees_coin.amount) return Err.BAD_COINBASE_REWARD # Verify that removed coin puzzle_hashes match with calculated puzzle_hashes diff --git a/src/coin_store.py b/src/full_node/coin_store.py similarity index 99% rename from src/coin_store.py rename to src/full_node/coin_store.py index d5b7a5da..4950ad65 100644 --- a/src/coin_store.py +++ b/src/full_node/coin_store.py @@ -3,8 +3,8 @@ from typing import Dict, Optional, List from pathlib import Path import aiosqlite from src.types.full_block import FullBlock -from src.types.hashable.Coin import Coin -from src.types.hashable.CoinRecord import CoinRecord +from src.types.hashable.coin import Coin +from src.types.hashable.coin_record import CoinRecord from src.types.sized_bytes import bytes32 from src.types.header import Header from src.util.ints import uint32 diff --git a/src/full_node.py b/src/full_node/full_node.py similarity index 94% rename from src/full_node.py rename to src/full_node/full_node.py index a568899a..3fa01eaa 100644 --- a/src/full_node.py +++ b/src/full_node/full_node.py @@ -3,38 +3,39 @@ import concurrent import logging import time from asyncio import Event -from secrets import token_bytes from typing import AsyncGenerator, List, Optional, Tuple, Dict +from chiabip158 import PyBIP158 from chiapos import Verifier import src.protocols.wallet_protocol -from src.blockchain import Blockchain, ReceiveBlockResult +from src.full_node.blockchain import Blockchain, ReceiveBlockResult from src.consensus.block_rewards import calculate_base_fee -from src.consensus.constants import constants +from src.consensus.constants import constants as consensus_constants from src.consensus.pot_iterations import calculate_iterations from src.consensus.weight_verifier import verify_weight from src.protocols.wallet_protocol import FullProofForHash, ProofHash -from src.store import FullNodeStore +from src.full_node.store import FullNodeStore from src.protocols import farmer_protocol, full_node_protocol, timelord_protocol +from src.util.merkle_set import MerkleSet from src.util.bundle_tools import best_solution_program -from src.mempool_manager import MempoolManager +from src.full_node.mempool_manager import MempoolManager from src.server.outbound_message import Delivery, Message, NodeType, OutboundMessage from src.server.server import ChiaServer from src.types.body import Body from src.types.challenge import Challenge from src.types.full_block import FullBlock -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin, hash_coin_list from src.types.hashable.BLSSignature import BLSSignature from src.util.hash import std_hash -from src.types.hashable.SpendBundle import SpendBundle -from src.types.hashable.Program import Program +from src.types.hashable.spend_bundle import SpendBundle +from src.types.hashable.program import Program from src.types.header import Header, HeaderData from src.types.header_block import HeaderBlock from src.types.peer_info import PeerInfo from src.types.proof_of_space import ProofOfSpace from src.types.sized_bytes import bytes32 -from src.coin_store import CoinStore +from src.full_node.coin_store import CoinStore from src.util import errors from src.util.api_decorators import api_request from src.util.errors import InvalidUnfinishedBlock @@ -52,7 +53,9 @@ class FullNode: mempool_manager: MempoolManager, unspent_store: CoinStore, name: str = None, + override_constants={}, ): + self.config: Dict = config self.store: FullNodeStore = store self.blockchain: Blockchain = blockchain @@ -60,6 +63,9 @@ class FullNode: self._shut_down = False # Set to true to close all infinite loops self.server: Optional[ChiaServer] = None self.unspent_store: CoinStore = unspent_store + self.constants = consensus_constants.copy() + for key, value in override_constants.items(): + self.constants[key] = value if name: self.log = logging.getLogger(name) else: @@ -934,11 +940,6 @@ class FullNode: ): return assert challenge is not None - print(self.store.unfinished_blocks.keys()) - print( - "It's none..", - (challenge.get_hash(), new_unfinished_block.number_of_iterations), - ) yield OutboundMessage( NodeType.FULL_NODE, Message( @@ -1030,7 +1031,7 @@ class FullNode: int(iterations_needed / (self.store.get_proof_of_time_estimate_ips())) ) - if expected_time > constants["PROPAGATION_DELAY_THRESHOLD"]: + if expected_time > self.constants["PROPAGATION_DELAY_THRESHOLD"]: self.log.info(f"Block is slow, expected {expected_time} seconds, waiting") # If this block is slow, sleep to allow faster blocks to come out first await asyncio.sleep(5) @@ -1043,7 +1044,7 @@ class FullNode: # If this is the first block we see at this height, propagate self.store.set_unfinished_block_leader((block.height, expected_time)) elif block.height == leader[0]: - if expected_time > leader[1] + constants["PROPAGATION_THRESHOLD"]: + if expected_time > leader[1] + self.constants["PROPAGATION_THRESHOLD"]: # If VDF is expected to finish X seconds later than the best, don't propagate self.log.info( f"VDF will finish too late {expected_time} seconds, so don't propagate" @@ -1245,8 +1246,21 @@ class FullNode: prev_header_hash: bytes32 = target_tip.get_hash() timestamp: uint64 = uint64(int(time.time())) - # TODO(straya): use a real BIP158 filter based on transactions - filter_hash: bytes32 = token_bytes(32) + # Create filter + byte_array_tx: List[bytes32] = [] + if spend_bundle: + additions: List[Coin] = spend_bundle.additions() + removals: List[Coin] = spend_bundle.removals() + for coin in additions: + byte_array_tx.append(bytearray(coin.puzzle_hash)) + for coin in removals: + byte_array_tx.append(bytearray(coin.name())) + byte_array_tx.append(bytearray(request.coinbase.puzzle_hash)) + byte_array_tx.append(bytearray(fees_coin.puzzle_hash)) + + bip158: PyBIP158 = PyBIP158(byte_array_tx) + encoded_filter = bytes(bip158.GetEncoded()) + proof_of_space_hash: bytes32 = request.proof_of_space.get_hash() body_hash: Body = body.get_hash() difficulty = self.blockchain.get_next_difficulty(target_tip.header_hash) @@ -1255,16 +1269,50 @@ class FullNode: vdf_ips: uint64 = self.blockchain.get_next_ips(target_tip_block) iterations_needed: uint64 = calculate_iterations( - request.proof_of_space, difficulty, vdf_ips, constants["MIN_BLOCK_TIME"], + request.proof_of_space, + difficulty, + vdf_ips, + self.constants["MIN_BLOCK_TIME"], ) - additions_root = token_bytes(32) # TODO(straya) - removal_root = token_bytes(32) # TODO(straya) + + removal_merkle_set = MerkleSet() + addition_merkle_set = MerkleSet() + + additions = [] + removals = [] + + if spend_bundle: + additions = spend_bundle.additions() + removals = spend_bundle.removals() + + additions.append(request.coinbase) + additions.append(fees_coin) + + # Create removal Merkle set + for coin in removals: + removal_merkle_set.add_already_hashed(coin.name()) + + # Create addition Merkle set + puzzlehash_coins_map: Dict[bytes32, List[Coin]] = {} + for coin in additions: + if coin.puzzle_hash in puzzlehash_coins_map: + puzzlehash_coins_map[coin.puzzle_hash].append(coin) + else: + puzzlehash_coins_map[coin.puzzle_hash] = [coin] + + # Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash + for puzzle, coins in puzzlehash_coins_map.items(): + addition_merkle_set.add_already_hashed(puzzle) + addition_merkle_set.add_already_hashed(hash_coin_list(coins)) + + additions_root = addition_merkle_set.get_root() + removal_root = removal_merkle_set.get_root() block_header_data: HeaderData = HeaderData( uint32(target_tip.height + 1), prev_header_hash, timestamp, - filter_hash, + encoded_filter, proof_of_space_hash, body_hash, target_tip.weight + difficulty, @@ -1648,3 +1696,25 @@ class FullNode: yield OutboundMessage( NodeType.WALLET, Message("full_proof_for_hash", proof), Delivery.RESPOND ) + + @api_request + async def request_additions( + self, request: src.protocols.wallet_protocol.RequestAdditions + ) -> OutboundMessageGenerator: + block: Optional[FullBlock] = await self.store.get_block(request.header_hash) + if block: + additions = block.additions() + response = src.protocols.wallet_protocol.Additions( + block.height, block.header_hash, additions + ) + yield OutboundMessage( + NodeType.WALLET, + Message("response_additions", response), + Delivery.BROADCAST, + ) + else: + yield OutboundMessage( + NodeType.WALLET, + Message("response_reject_additions", request), + Delivery.BROADCAST, + ) diff --git a/src/mempool.py b/src/full_node/mempool.py similarity index 98% rename from src/mempool.py rename to src/full_node/mempool.py index ed158ef4..c4223943 100644 --- a/src/mempool.py +++ b/src/full_node/mempool.py @@ -2,7 +2,7 @@ from typing import List, Dict from sortedcontainers import SortedDict -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.mempool_item import MempoolItem from src.types.sized_bytes import bytes32 from src.util.ints import uint32, uint64 diff --git a/src/mempool_manager.py b/src/full_node/mempool_manager.py similarity index 96% rename from src/mempool_manager.py rename to src/full_node/mempool_manager.py index 9afe1dfd..e856b890 100644 --- a/src/mempool_manager.py +++ b/src/full_node/mempool_manager.py @@ -5,14 +5,14 @@ import logging from src.consensus.constants import constants as consensus_constants from src.util.bundle_tools import best_solution_program from src.types.full_block import FullBlock -from src.types.hashable.Coin import Coin -from src.types.hashable.SpendBundle import SpendBundle -from src.types.hashable.CoinRecord import CoinRecord +from src.types.hashable.coin import Coin +from src.types.hashable.spend_bundle import SpendBundle +from src.types.hashable.coin_record import CoinRecord from src.types.header import Header from src.types.mempool_item import MempoolItem -from src.mempool import Mempool +from src.full_node.mempool import Mempool from src.types.sized_bytes import bytes32 -from src.coin_store import CoinStore +from src.full_node.coin_store import CoinStore from src.util.ConsensusError import Err from src.util.mempool_check_conditions import ( get_name_puzzle_conditions, @@ -29,7 +29,7 @@ log = logging.getLogger(__name__) class MempoolManager: def __init__(self, unspent_store: CoinStore, override_constants: Dict = {}): # Allow passing in custom overrides - self.constants: Dict = consensus_constants + self.constants: Dict = consensus_constants.copy() for key, value in override_constants.items(): self.constants[key] = value @@ -44,9 +44,9 @@ class MempoolManager: self.old_mempools: SortedDict[uint32, Dict[bytes32, MempoolItem]] = SortedDict() self.unspent_store = unspent_store - tx_per_sec = consensus_constants["TX_PER_SEC"] - sec_per_block = consensus_constants["BLOCK_TIME_TARGET"] - block_buffer_count = consensus_constants["MEMPOOL_BLOCK_BUFFER"] + tx_per_sec = self.constants["TX_PER_SEC"] + sec_per_block = self.constants["BLOCK_TIME_TARGET"] + block_buffer_count = self.constants["MEMPOOL_BLOCK_BUFFER"] # MEMPOOL_SIZE = 60000 self.mempool_size = tx_per_sec * sec_per_block * block_buffer_count @@ -101,7 +101,7 @@ class MempoolManager: # Calculate the cost and fees program = best_solution_program(new_spend) # npc contains names of the coins removed, puzzle_hashes and their spend conditions - fail_reason, npc_list, cost = await get_name_puzzle_conditions(program) + fail_reason, npc_list, cost = get_name_puzzle_conditions(program) if fail_reason: return None, fail_reason @@ -117,7 +117,7 @@ class MempoolManager: # Check additions for max coin amount for coin in additions: - if coin.amount >= consensus_constants["MAX_COIN_AMOUNT"]: + if coin.amount >= self.constants["MAX_COIN_AMOUNT"]: return None, Err.COIN_AMOUNT_EXCEEDS_MAXIMUM # Watch out for duplicate outputs diff --git a/src/store.py b/src/full_node/store.py similarity index 100% rename from src/store.py rename to src/full_node/store.py diff --git a/src/pool/__init__.py b/src/pool/__init__.py index 13f0e29a..1acfd74a 100644 --- a/src/pool/__init__.py +++ b/src/pool/__init__.py @@ -2,7 +2,7 @@ import blspy from src.types.sized_bytes import bytes32 from src.util.ints import uint64 -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.hashable.BLSSignature import BLSSignature, BLSPublicKey from src.wallet.puzzles.p2_delegated_puzzle import puzzle_for_pk diff --git a/src/protocols/farmer_protocol.py b/src/protocols/farmer_protocol.py index 3d4cf677..07ae7f28 100644 --- a/src/protocols/farmer_protocol.py +++ b/src/protocols/farmer_protocol.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from blspy import PrependSignature -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.hashable.BLSSignature import BLSSignature from src.types.proof_of_space import ProofOfSpace from src.types.sized_bytes import bytes32 diff --git a/src/protocols/full_node_protocol.py b/src/protocols/full_node_protocol.py index 318e0ac4..4b2a5790 100644 --- a/src/protocols/full_node_protocol.py +++ b/src/protocols/full_node_protocol.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from typing import List from src.types.full_block import FullBlock -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.spend_bundle import SpendBundle from src.types.header_block import HeaderBlock from src.types.peer_info import PeerInfo from src.types.proof_of_time import ProofOfTime diff --git a/src/protocols/shared_protocol.py b/src/protocols/shared_protocol.py index 2a9742cc..387d03a1 100644 --- a/src/protocols/shared_protocol.py +++ b/src/protocols/shared_protocol.py @@ -5,7 +5,7 @@ from src.types.sized_bytes import bytes32 from src.util.cbor_message import cbor_message from src.util.ints import uint16 -protocol_version = "0.0.5" +protocol_version = "0.0.6" """ Handshake when establishing a connection between two servers. diff --git a/src/protocols/wallet_protocol.py b/src/protocols/wallet_protocol.py index f849ffc3..55d3a79c 100644 --- a/src/protocols/wallet_protocol.py +++ b/src/protocols/wallet_protocol.py @@ -2,7 +2,8 @@ from dataclasses import dataclass from typing import List, Tuple from src.types.body import Body -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.coin import Coin +from src.types.hashable.spend_bundle import SpendBundle from src.types.header_block import HeaderBlock from src.types.sized_bytes import bytes32 from src.util.cbor_message import cbor_message @@ -86,3 +87,33 @@ class FullProofForHash: @cbor_message class ProofHash: proof_hash: bytes32 + + +@dataclass(frozen=True) +@cbor_message +class RequestAdditions: + height: uint32 + header_hash: bytes32 + + +@dataclass(frozen=True) +@cbor_message +class RequestRemovals: + height: uint32 + header_hash: bytes32 + + +@dataclass(frozen=True) +@cbor_message +class Additions: + height: uint32 + header_hash: bytes32 + coins: List[Coin] + + +@dataclass(frozen=True) +@cbor_message +class Removals: + height: uint32 + header_hash: bytes32 + coins: List[Coin] diff --git a/src/rpc/rpc_client.py b/src/rpc/rpc_client.py index e77bce1c..cd41bd46 100644 --- a/src/rpc/rpc_client.py +++ b/src/rpc/rpc_client.py @@ -7,7 +7,7 @@ from src.types.full_block import FullBlock from src.types.header import Header from src.types.sized_bytes import bytes32 from src.util.ints import uint16 -from src.types.hashable.CoinRecord import CoinRecord +from src.types.hashable.coin_record import CoinRecord class RpcClient: diff --git a/src/rpc/rpc_server.py b/src/rpc/rpc_server.py index d7d0076e..b230efec 100644 --- a/src/rpc/rpc_server.py +++ b/src/rpc/rpc_server.py @@ -5,7 +5,7 @@ from typing import Any, Callable, List, Optional from aiohttp import web -from src.full_node import FullNode +from src.full_node.full_node import FullNode from src.types.header import Header from src.types.full_block import FullBlock from src.types.peer_info import PeerInfo diff --git a/src/server/start_full_node.py b/src/server/start_full_node.py index 7a89c9f8..ab4a5cd5 100644 --- a/src/server/start_full_node.py +++ b/src/server/start_full_node.py @@ -11,17 +11,17 @@ try: except ImportError: uvloop = None -from src.blockchain import Blockchain +from src.full_node.blockchain import Blockchain from src.consensus.constants import constants -from src.store import FullNodeStore -from src.full_node import FullNode +from src.full_node.store import FullNodeStore +from src.full_node.full_node import FullNode from src.rpc.rpc_server import start_rpc_server -from src.mempool_manager import MempoolManager +from src.full_node.mempool_manager import MempoolManager from src.server.server import ChiaServer from src.server.connection import NodeType from src.types.full_block import FullBlock from src.types.peer_info import PeerInfo -from src.coin_store import CoinStore +from src.full_node.coin_store import CoinStore from src.util.logging import initialize_logging from src.util.config import load_config_cli from setproctitle import setproctitle diff --git a/src/server/start_wallet.py b/src/server/start_wallet.py index 6c867a5a..dae5460f 100644 --- a/src/server/start_wallet.py +++ b/src/server/start_wallet.py @@ -2,7 +2,7 @@ import asyncio import signal import logging -from src.wallet.wallet import Wallet +from src.wallet.wallet_node import WalletNode try: import uvloop @@ -29,7 +29,7 @@ async def main(): log = logging.getLogger(__name__) setproctitle("Chia_Wallet") - wallet = await Wallet.create(config, key_config) + wallet = await WalletNode.create(config, key_config) full_node_peer = PeerInfo( config["full_node_peer"]["host"], config["full_node_peer"]["port"] diff --git a/src/timelord.py b/src/timelord.py index 1c3b99d8..c3d1dd5b 100644 --- a/src/timelord.py +++ b/src/timelord.py @@ -19,8 +19,11 @@ log = logging.getLogger(__name__) class Timelord: - def __init__(self, config: Dict): + def __init__( + self, config: Dict, discrimant_size_bits=constants["DISCRIMINANT_SIZE_BITS"] + ): self.config: Dict = config + self.discriminant_size_bits = discrimant_size_bits self.free_servers: List[Tuple[str, str]] = list( zip(self.config["vdf_server_ips"], self.config["vdf_server_ports"]) ) @@ -202,9 +205,7 @@ class Timelord: async def _do_process_communication( self, challenge_hash, challenge_weight, ip, port ): - disc: int = create_discriminant( - challenge_hash, constants["DISCRIMINANT_SIZE_BITS"] - ) + disc: int = create_discriminant(challenge_hash, self.discriminant_size_bits) log.info("Attempting SSH connection") proc = await asyncio.create_subprocess_shell( @@ -310,7 +311,7 @@ class Timelord: self.config["n_wesolowski"], proof_bytes, ) - if not proof_of_time.is_valid(constants["DISCRIMINANT_SIZE_BITS"]): + if not proof_of_time.is_valid(self.discriminant_size_bits): log.error("Invalid proof of time") response = timelord_protocol.ProofOfTimeFinished(proof_of_time) diff --git a/src/types/body.py b/src/types/body.py index 937a3c21..b5fea50c 100644 --- a/src/types/body.py +++ b/src/types/body.py @@ -2,8 +2,8 @@ from dataclasses import dataclass from typing import Optional from src.types.hashable.BLSSignature import BLSSignature -from src.types.hashable.Program import Program -from src.types.hashable.Coin import Coin +from src.types.hashable.program import Program +from src.types.hashable.coin import Coin from src.util.ints import uint64 from src.util.streamable import Streamable, streamable from src.types.sized_bytes import bytes32 diff --git a/src/types/ConditionVarPair.py b/src/types/condition_var_pair.py similarity index 100% rename from src/types/ConditionVarPair.py rename to src/types/condition_var_pair.py diff --git a/src/types/full_block.py b/src/types/full_block.py index ac89f095..4057119c 100644 --- a/src/types/full_block.py +++ b/src/types/full_block.py @@ -3,7 +3,7 @@ from typing import Tuple, List, Optional from src.types.name_puzzle_condition import NPC from src.types.body import Body -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.header import Header from src.types.sized_bytes import bytes32 from src.util.mempool_check_conditions import get_name_puzzle_conditions @@ -50,6 +50,21 @@ class FullBlock(Streamable): def header_hash(self) -> bytes32: return self.header.header_hash + def additions(self) -> List[Coin]: + additions: List[Coin] = [] + + if self.body.transactions is not None: + # This should never throw here, block must be valid if it comes to here + err, npc_list, cost = get_name_puzzle_conditions(self.body.transactions) + # created coins + if npc_list is not None: + additions.extend(additions_for_npc(npc_list)) + + additions.append(self.body.coinbase) + additions.append(self.body.fees_coin) + + return additions + async def tx_removals_and_additions(self) -> Tuple[List[bytes32], List[Coin]]: """ Doesn't return coinbase and fee reward. @@ -60,11 +75,8 @@ class FullBlock(Streamable): additions: List[Coin] = [] if self.body.transactions is not None: - # ensure block program generates solutions # This should never throw here, block must be valid if it comes to here - err, npc_list, cost = await get_name_puzzle_conditions( - self.body.transactions - ) + err, npc_list, cost = get_name_puzzle_conditions(self.body.transactions) # build removals list if npc_list is None: return [], [] diff --git a/src/types/hashable/Coin.py b/src/types/hashable/Coin.py index 2ca6c952..69219065 100644 --- a/src/types/hashable/Coin.py +++ b/src/types/hashable/Coin.py @@ -1,9 +1,11 @@ import io from dataclasses import dataclass +from typing import List from clvm.casts import int_to_bytes, int_from_bytes from src.types.sized_bytes import bytes32 +from src.util.hash import std_hash from src.util.ints import uint64 from src.util.streamable import streamable, Streamable @@ -22,6 +24,10 @@ class Coin(Streamable): def name(self) -> bytes32: return self.get_hash() + @property + def name_str(self) -> str: + return self.name().hex() + @classmethod def from_bytes(cls, blob): parent_coin_info = blob[:32] @@ -35,3 +41,13 @@ class Coin(Streamable): f.write(self.puzzle_hash) f.write(int_to_bytes(self.amount)) return f.getvalue() + + +def hash_coin_list(coin_list: List[Coin]) -> bytes32: + coin_list.sort(key=lambda x: x.name_str, reverse=True) + buffer = bytearray() + + for coin in coin_list: + buffer.extend(coin.name()) + + return std_hash(buffer) diff --git a/src/types/hashable/Program.py b/src/types/hashable/Program.py index 098dfc60..4d985512 100644 --- a/src/types/hashable/Program.py +++ b/src/types/hashable/Program.py @@ -44,5 +44,4 @@ class Program(SExp): # type: ignore # noqa return bytes(self).hex() def get_hash(self) -> bytes32: - # print("Bytes self", bytes(self)) return bytes32(std_hash(bytes(self))) diff --git a/src/types/hashable/CoinRecord.py b/src/types/hashable/coin_record.py similarity index 93% rename from src/types/hashable/CoinRecord.py rename to src/types/hashable/coin_record.py index a5ade0c7..18340323 100644 --- a/src/types/hashable/CoinRecord.py +++ b/src/types/hashable/coin_record.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.sized_bytes import bytes32 from src.util.streamable import Streamable, streamable from src.util.ints import uint32 diff --git a/src/types/hashable/CoinSolution.py b/src/types/hashable/coin_solution.py similarity index 89% rename from src/types/hashable/CoinSolution.py rename to src/types/hashable/coin_solution.py index d8eeb559..1dc6666b 100644 --- a/src/types/hashable/CoinSolution.py +++ b/src/types/hashable/coin_solution.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from .Coin import Coin -from .Program import Program +from .coin import Coin +from .program import Program from src.util.streamable import Streamable, streamable diff --git a/src/types/hashable/SpendBundle.py b/src/types/hashable/spend_bundle.py similarity index 96% rename from src/types/hashable/SpendBundle.py rename to src/types/hashable/spend_bundle.py index eebf7c24..affb6824 100644 --- a/src/types/hashable/SpendBundle.py +++ b/src/types/hashable/spend_bundle.py @@ -1,12 +1,12 @@ from dataclasses import dataclass from typing import List, Dict -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.sized_bytes import bytes32 from src.util.chain_utils import additions_for_solution from src.util.streamable import Streamable, streamable from .BLSSignature import BLSSignature -from .CoinSolution import CoinSolution +from .coin_solution import CoinSolution @dataclass(frozen=True) diff --git a/src/types/header.py b/src/types/header.py index 1e726e25..c5dcdfd7 100644 --- a/src/types/header.py +++ b/src/types/header.py @@ -13,7 +13,7 @@ class HeaderData(Streamable): height: uint32 prev_header_hash: bytes32 timestamp: uint64 - filter_hash: bytes32 + filter: bytes proof_of_space_hash: bytes32 body_hash: bytes32 weight: uint64 diff --git a/src/types/mempool_item.py b/src/types/mempool_item.py index 345637b8..799f2adb 100644 --- a/src/types/mempool_item.py +++ b/src/types/mempool_item.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.spend_bundle import SpendBundle from src.types.sized_bytes import bytes32 from src.util.ints import uint64 diff --git a/src/types/name_puzzle_condition.py b/src/types/name_puzzle_condition.py index aa2333e9..619ea5dd 100644 --- a/src/types/name_puzzle_condition.py +++ b/src/types/name_puzzle_condition.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Dict, List -from src.types.ConditionVarPair import ConditionVarPair +from src.types.condition_var_pair import ConditionVarPair from src.types.sized_bytes import bytes32 from src.util.condition_tools import ConditionOpcode diff --git a/src/ui/prompt_ui.py b/src/ui/prompt_ui.py index fc3181a8..e6531af9 100644 --- a/src/ui/prompt_ui.py +++ b/src/ui/prompt_ui.py @@ -331,7 +331,7 @@ class FullNodeUI: else: self.syncing.text = f"Syncing" else: - self.syncing.text = "Not syncing" + self.syncing.text = "Synced" total_iters = self.lca_block.data.total_iters diff --git a/src/util/blockchain_check_conditions.py b/src/util/blockchain_check_conditions.py index 9baf54ca..e1cde636 100644 --- a/src/util/blockchain_check_conditions.py +++ b/src/util/blockchain_check_conditions.py @@ -3,8 +3,8 @@ from typing import Optional, Dict, List from clvm.casts import int_from_bytes -from src.types.ConditionVarPair import ConditionVarPair -from src.types.hashable.CoinRecord import CoinRecord +from src.types.condition_var_pair import ConditionVarPair +from src.types.hashable.coin_record import CoinRecord from src.types.header import Header from src.types.sized_bytes import bytes32 from src.util.condition_tools import ConditionOpcode diff --git a/src/util/bundle_tools.py b/src/util/bundle_tools.py index b4f6b895..eafb56de 100644 --- a/src/util/bundle_tools.py +++ b/src/util/bundle_tools.py @@ -1,7 +1,7 @@ from clvm_tools import binutils -from src.types.hashable.Program import Program -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.program import Program +from src.types.hashable.spend_bundle import SpendBundle def best_solution_program(bundle: SpendBundle) -> Program: diff --git a/src/util/chain_utils.py b/src/util/chain_utils.py index cdfb9448..0fa696a7 100644 --- a/src/util/chain_utils.py +++ b/src/util/chain_utils.py @@ -1,6 +1,6 @@ from typing import List -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.util.condition_tools import ( created_outputs_for_conditions_dict, conditions_dict_for_solution, diff --git a/src/util/condition_tools.py b/src/util/condition_tools.py index 73142b9e..eb33fd3d 100644 --- a/src/util/condition_tools.py +++ b/src/util/condition_tools.py @@ -6,11 +6,11 @@ from clvm.EvalError import EvalError from clvm.casts import int_from_bytes from clvm.subclass_sexp import BaseSExp -from src.types.ConditionVarPair import ConditionVarPair +from src.types.condition_var_pair import ConditionVarPair from src.types.condition_opcodes import ConditionOpcode from src.types.hashable.BLSSignature import BLSSignature, BLSPublicKey -from src.types.hashable.Coin import Coin -from src.types.hashable.Program import Program +from src.types.hashable.coin import Coin +from src.types.hashable.program import Program from src.types.sized_bytes import bytes32 from .ConsensusError import Err, ConsensusError diff --git a/src/util/mempool_check_conditions.py b/src/util/mempool_check_conditions.py index c7814d8c..b01ea66f 100644 --- a/src/util/mempool_check_conditions.py +++ b/src/util/mempool_check_conditions.py @@ -4,12 +4,12 @@ import clvm from clvm import EvalError from clvm.casts import int_from_bytes -from src.types.ConditionVarPair import ConditionVarPair -from src.types.hashable.Program import Program -from src.types.hashable.SpendBundle import SpendBundle -from src.types.hashable.CoinRecord import CoinRecord +from src.types.condition_var_pair import ConditionVarPair +from src.types.hashable.program import Program +from src.types.hashable.spend_bundle import SpendBundle +from src.types.hashable.coin_record import CoinRecord from src.types.name_puzzle_condition import NPC -from src.mempool import Mempool +from src.full_node.mempool import Mempool from src.types.sized_bytes import bytes32 from src.util.condition_tools import ConditionOpcode, conditions_dict_for_solution from src.util.ConsensusError import Err @@ -91,7 +91,7 @@ def mempool_assert_time_exceeds(condition: ConditionVarPair): return None -async def get_name_puzzle_conditions( +def get_name_puzzle_conditions( block_program: Program, ) -> Tuple[Optional[Err], List[NPC], int]: """ diff --git a/src/util/merkle_set.py b/src/util/merkle_set.py new file mode 100644 index 00000000..2fbf7260 --- /dev/null +++ b/src/util/merkle_set.py @@ -0,0 +1,357 @@ +from hashlib import blake2b, sha256 +from typing import Dict, List + +""" +A simple, confidence-inspiring Merkle Set standard + +Advantages of this standard: +Low CPU requirements +Small proofs of inclusion/exclusion +Reasonably simple implementation + +The main tricks in this standard are: + +Uses blake2b because that has the best performance on 512 bit inputs +Skips repeated hashing of exactly two things even when they share prefix bits + + +Proofs support proving including/exclusion for a large number of values in +a single string. They're a serialization of a subset of the tree. + +Proof format: + +multiproof: subtree +subtree: middle or terminal or truncated or empty +middle: MIDDLE 1 subtree subtree +terminal: TERMINAL 1 hash 32 +# If the sibling is empty truncated implies more than two children. +truncated: TRUNCATED 1 hash 32 +empty: EMPTY 1 +EMPTY: \x00 +TERMINAL: \x01 +MIDDLE: \x02 +TRUNCATED: \x03 +""" + +EMPTY = bytes([0]) +TERMINAL = bytes([1]) +MIDDLE = bytes([2]) +TRUNCATED = bytes([3]) + +BLANK = bytes([0] * 32) + +prehashed: Dict = {} + + +def init_prehashed(): + for x in [EMPTY, TERMINAL, MIDDLE]: + for y in [EMPTY, TERMINAL, MIDDLE]: + prehashed[x + y] = blake2b(bytes([0] * 30) + x + y) + + +init_prehashed() + + +def hashdown(mystr): + assert len(mystr) == 66 + h = prehashed[bytes(mystr[0:1] + mystr[33:34])].copy() + h.update(mystr[1:33] + mystr[34:]) + return h.digest()[:32] + + +def compress_root(mystr): + assert len(mystr) == 33 + if mystr[0:1] == MIDDLE: + return mystr[1:] + if mystr[0:1] == EMPTY: + assert mystr[1:] == BLANK + return BLANK + return blake2b(mystr).digest()[:32] + + +def get_bit(mybytes, pos): + assert len(mybytes) == 32 + return (mybytes[pos // 8] >> (7 - (pos % 8))) & 1 + + +class MerkleSet: + def __init__(self, root=None): + self.root = root + if root is None: + self.root = _empty + + def get_root(self): + return compress_root(self.root.get_hash()) + + def add_already_hashed(self, toadd): + self.root = self.root.add(toadd, 0) + + def remove_already_hashed(self, toremove): + self.root = self.root.remove(toremove, 0) + + def is_included_already_hashed(self, tocheck): + proof: List = [] + r = self.root.is_included(tocheck, 0, proof) + return r, b"".join(proof) + + def _audit(self, hashes): + newhashes: List = [] + self.root._audit(newhashes, []) + assert newhashes == sorted(newhashes) + + +class EmptyNode: + def __init__(self): + self.hash = BLANK + + def get_hash(self): + return EMPTY + BLANK + + def is_empty(self): + return True + + def is_terminal(self): + return False + + def is_double(self): + raise SetError() + + def add(self, toadd, depth): + return TerminalNode(toadd) + + def remove(self, toremove, depth): + return self + + def is_included(self, tocheck, depth, p): + p.append(EMPTY) + return False + + def other_included(self, tocheck, depth, p, collapse): + p.append(EMPTY) + + def _audit(self, hashes, bits): + pass + + +_empty = EmptyNode() + + +class TerminalNode: + def __init__(self, hash, bits=None): + assert len(hash) == 32 + self.hash = hash + if bits is not None: + self._audit([], bits) + + def get_hash(self): + return TERMINAL + self.hash + + def is_empty(self): + return False + + def is_terminal(self): + return True + + def is_double(self): + raise SetError() + + def add(self, toadd, depth): + if toadd == self.hash: + return self + if toadd > self.hash: + return self._make_middle([self, TerminalNode(toadd)], depth) + else: + return self._make_middle([TerminalNode(toadd), self], depth) + + def _make_middle(self, children, depth): + cbits = [get_bit(child.hash, depth) for child in children] + if cbits[0] != cbits[1]: + return MiddleNode(children) + nextvals = [None, None] + nextvals[cbits[0] ^ 1] = _empty # type: ignore + nextvals[cbits[0]] = self._make_middle(children, depth + 1) + return MiddleNode(nextvals) + + def remove(self, toremove, depth): + if toremove == self.hash: + return _empty + return self + + def is_included(self, tocheck, depth, proof): + proof.append(TERMINAL + self.hash) + return tocheck == self.hash + + def other_included(self, tocheck, depth, p, collapse): + p.append(TERMINAL + self.hash) + + def _audit(self, hashes, bits): + hashes.append(self.hash) + for pos, v in enumerate(bits): + assert get_bit(self.hash, pos) == v + + +class MiddleNode: + def __init__(self, children): + self.children = children + if children[0].is_empty() and children[1].is_double(): + self.hash = children[1].hash + elif children[1].is_empty() and children[0].is_double(): + self.hash = children[0].hash + else: + if children[0].is_empty() and ( + children[1].is_empty() or children[1].is_terminal() + ): + raise SetError() + if children[1].is_empty() and children[0].is_terminal(): + raise SetError + if ( + children[0].is_terminal() + and children[1].is_terminal() + and children[0].hash >= children[1].hash + ): + raise SetError + self.hash = hashdown(children[0].get_hash() + children[1].get_hash()) + + def get_hash(self): + return MIDDLE + self.hash + + def is_empty(self): + return False + + def is_terminal(self): + return False + + def is_double(self): + if self.children[0].is_empty(): + return self.children[1].is_double() + if self.children[1].is_empty(): + return self.children[0].is_double() + return self.children[0].is_terminal() and self.children[1].is_terminal() + + def add(self, toadd, depth): + bit = get_bit(toadd, depth) + child = self.children[bit] + newchild = child.add(toadd, depth + 1) + if newchild is child: + return self + newvals = [x for x in self.children] + newvals[bit] = newchild + return MiddleNode(newvals) + + def remove(self, toremove, depth): + bit = get_bit(toremove, depth) + child = self.children[bit] + newchild = child.remove(toremove, depth + 1) + if newchild is child: + return self + otherchild = self.children[bit ^ 1] + if newchild.is_empty() and otherchild.is_terminal(): + return otherchild + if newchild.is_terminal() and otherchild.is_empty(): + return newchild + newvals = [x for x in self.children] + newvals[bit] = newchild + return MiddleNode(newvals) + + def is_included(self, tocheck, depth, p): + p.append(MIDDLE) + if get_bit(tocheck, depth) == 0: + r = self.children[0].is_included(tocheck, depth + 1, p) + self.children[1].other_included( + tocheck, depth + 1, p, not self.children[0].is_empty() + ) + return r + else: + self.children[0].other_included( + tocheck, depth + 1, p, not self.children[1].is_empty() + ) + return self.children[1].is_included(tocheck, depth + 1, p) + + def other_included(self, tocheck, depth, p, collapse): + if collapse or not self.is_double(): + p.append(TRUNCATED + self.hash) + else: + self.is_included(tocheck, depth, p) + + def _audit(self, hashes, bits): + self.children[0]._audit(hashes, bits + [0]) + self.children[1]._audit(hashes, bits + [1]) + + +class TruncatedNode: + def __init__(self, hash): + self.hash = hash + + def get_hash(self): + return MIDDLE + self.hash + + def is_empty(self): + return False + + def is_terminal(self): + return False + + def is_double(self): + return False + + def is_included(self, tocheck, depth, p): + raise SetError() + + def other_included(self, tocheck, depth, p, collapse): + p.append(TRUNCATED + self.hash) + + +class SetError(BaseException): + pass + + +def confirm_included(root, val, proof): + return confirm_not_included_already_hashed(root, sha256(val).digest(), proof) + + +def confirm_included_already_hashed(root, val, proof): + return _confirm(root, val, proof, True) + + +def confirm_not_included(root, val, proof): + return confirm_not_included_already_hashed(root, sha256(val).digest(), proof) + + +def confirm_not_included_already_hashed(root, val, proof): + return _confirm(root, val, proof, False) + + +def _confirm(root, val, proof, expected): + try: + p = deserialize_proof(proof) + if p.get_root() != root: + return False + r, junk = p.is_included_already_hashed(val) + return r == expected + except SetError: + return False + + +def deserialize_proof(proof): + try: + r, pos = _deserialize(proof, 0, []) + if pos != len(proof): + raise SetError() + return MerkleSet(r) + except IndexError: + raise SetError() + + +def _deserialize(proof, pos, bits): + t = proof[pos : pos + 1] # flake8: noqa + if t == EMPTY: + return _empty, pos + 1 + if t == TERMINAL: + return TerminalNode(proof[pos + 1 : pos + 33], bits), pos + 33 + if t == TRUNCATED: + return TruncatedNode(proof[pos + 1 : pos + 33]), pos + 33 + if t != MIDDLE: + raise SetError() + v0, pos = _deserialize(proof, pos + 1, bits + [0]) + v1, pos = _deserialize(proof, pos, bits + [1]) + return MiddleNode([v0, v1]), pos diff --git a/src/util/streamable.py b/src/util/streamable.py index 97b33d4d..2b412fef 100644 --- a/src/util/streamable.py +++ b/src/util/streamable.py @@ -7,7 +7,7 @@ import pprint import json from typing import Any, BinaryIO, List, Type, get_type_hints, Union from src.util.byte_types import hexstr_to_bytes -from src.types.hashable.Program import Program +from src.types.hashable.program import Program from src.util.hash import std_hash from blspy import ( @@ -181,7 +181,7 @@ class Streamable: f.write(uint32(len(item)).to_bytes(4, "big")) f.write(item.encode("utf-8")) elif f_type is bool: - f.write(bytes(item)) + f.write(int(item).to_bytes(4, "big")) else: raise NotImplementedError(f"can't stream {item}, {f_type}") diff --git a/src/wallet/electron/README.md b/src/wallet/electron/README.md index c3524220..755bb292 100644 --- a/src/wallet/electron/README.md +++ b/src/wallet/electron/README.md @@ -1,9 +1,12 @@ -# Install -python3 -m venv .venv -. .venv/bin/activate +# Electron Wallet +## Install -pip install zerorpc -pip install pyinstaller +```npm install --runtime=electron --target=1.7.6``` -npm install --runtime=electron --target=1.7.6 -npm install electron-rebuild && ./node_modules/.bin/electron-rebuild +## Run: +```npm start``` + +## Error +If run fails because of electron try doing this + +```npm install electron-rebuild && ./node_modules/.bin/electron-rebuild``` diff --git a/src/wallet/electron/main.js b/src/wallet/electron/main.js index e1e0889d..392aa52e 100644 --- a/src/wallet/electron/main.js +++ b/src/wallet/electron/main.js @@ -9,7 +9,7 @@ const path = require('path') *************************************************************/ const PY_DIST_FOLDER = 'pydist' -const PY_FOLDER = 'wallet_rpc' +const PY_FOLDER = 'rpc' const PY_MODULE = 'rpc_wallet' // without .py suffix let pyProc = null diff --git a/src/wallet/electron/package-lock.json b/src/wallet/electron/package-lock.json index c36be3db..da026da0 100644 --- a/src/wallet/electron/package-lock.json +++ b/src/wallet/electron/package-lock.json @@ -177,15 +177,6 @@ "chainsaw": "~0.1.0" } }, - "bl": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", - "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -504,14 +495,6 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "requires": { - "mimic-response": "^1.0.0" - } - }, "decompress-zip": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/decompress-zip/-/decompress-zip-0.3.2.tgz", @@ -556,7 +539,8 @@ "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true }, "deep-is": { "version": "0.1.3", @@ -858,14 +842,6 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, "env-paths": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-1.0.0.tgz", @@ -998,16 +974,6 @@ "es5-ext": "~0.10.14" } }, - "event-lite": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/event-lite/-/event-lite-0.1.2.tgz", - "integrity": "sha512-HnSYx1BsJ87/p6swwzv+2v6B4X+uxUteoDfRxsAb1S1BePzQqOLevVmkdA15GHJVd9A9Ok6wygUR18Hu0YeV9g==" - }, - "expand-template": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz", - "integrity": "sha512-cebqLtV8KOZfw0UI8TEFWxtczxxC1jvyUvx6H4fyp1K1FN7A4Q+uggVUlOsI1K8AGU0rwOGqP8nCapdrw8CYQg==" - }, "ext": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", @@ -1094,11 +1060,6 @@ "mime-types": "^2.1.12" } }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "fs-extra": { "version": "0.30.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.30.0.tgz", @@ -1240,11 +1201,6 @@ "assert-plus": "^1.0.0" } }, - "github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" - }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -1356,18 +1312,14 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true }, "insert-css": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/insert-css/-/insert-css-2.0.0.tgz", "integrity": "sha1-610Ql7dUL0x56jBg067gfQU4gPQ=" }, - "int64-buffer": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/int64-buffer/-/int64-buffer-0.1.10.tgz", - "integrity": "sha1-J3siiofZWtd30HwTgyAiQGpHNCM=" - }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -1611,11 +1563,6 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" - }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1698,22 +1645,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "msgpack-lite": { - "version": "0.1.26", - "resolved": "https://registry.npmjs.org/msgpack-lite/-/msgpack-lite-0.1.26.tgz", - "integrity": "sha1-3TxQsm8FnyXn7e42REGDWOKprYk=", - "requires": { - "event-lite": "^0.1.1", - "ieee754": "^1.1.8", - "int64-buffer": "^0.1.9", - "isarray": "^1.0.0" - } - }, - "nan": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", - "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==" - }, "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", @@ -1761,11 +1692,6 @@ } } }, - "noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" - }, "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", @@ -2049,28 +1975,6 @@ "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==" }, - "prebuild-install": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-2.5.3.tgz", - "integrity": "sha512-/rI36cN2g7vDQnKWN8Uzupi++KjyqS9iS+/fpwG4Ea8d0Pip0PQ5bshUNzVwt+/D2MRfhVAplYMMvWLqWrCF/g==", - "requires": { - "detect-libc": "^1.0.3", - "expand-template": "^1.0.2", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "node-abi": "^2.2.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.1.6", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - } - }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -2137,15 +2041,6 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz", "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==" }, - "pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -2310,6 +2205,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -2505,21 +2401,6 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, - "simple-concat": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", - "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" - }, - "simple-get": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz", - "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==", - "requires": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, "single-line-log": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/single-line-log/-/single-line-log-1.1.2.tgz", @@ -2705,7 +2586,8 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true }, "sumchecker": { "version": "1.3.1", @@ -2739,42 +2621,6 @@ "yallist": "^3.0.3" } }, - "tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "requires": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - }, - "dependencies": { - "pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - } - } - }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - } - }, "throttleit": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-0.0.2.tgz", @@ -2835,11 +2681,6 @@ "os-tmpdir": "~1.0.1" } }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" - }, "touch": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/touch/-/touch-0.0.3.tgz", @@ -2926,11 +2767,6 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, - "underscore": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.3.3.tgz", - "integrity": "sha1-R6xTaD2vgyv6lS4XdEF9pHgXrkI=" - }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -3006,11 +2842,6 @@ "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" }, - "which-pm-runs": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", - "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=" - }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -3211,25 +3042,6 @@ "requires": { "fd-slicer": "~1.0.1" } - }, - "zeromq": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-4.6.0.tgz", - "integrity": "sha512-sU7pQqQj7f/C6orJZAXls+NEKaVMZZtnZqpMPTq5d5dP78CmdC0g15XIviFAN6poPuKl9qlGt74vipOUUuNeWg==", - "requires": { - "nan": "^2.6.2", - "prebuild-install": "^2.2.2" - } - }, - "zerorpc": { - "version": "git+https://github.com/0rpc/zerorpc-node.git#45d2e510a6dc236863ac9a2b92da0a2511ef54d9", - "from": "git+https://github.com/0rpc/zerorpc-node.git", - "requires": { - "msgpack-lite": "^0.1.26", - "underscore": "1.3.3", - "uuid": "^3.0.0", - "zeromq": "^4.6.0" - } } } } diff --git a/src/wallet/electron/package.json b/src/wallet/electron/package.json index 58337333..c0ea6577 100644 --- a/src/wallet/electron/package.json +++ b/src/wallet/electron/package.json @@ -16,11 +16,10 @@ "dialogs": "^2.0.1", "electron-rebuild": "^1.10.0", "jquery": "^3.4.1", - "qrcode": "^1.4.4", - "zerorpc": "git+https://github.com/0rpc/zerorpc-node.git" + "qrcode": "^1.4.4" }, "devDependencies": { - "electron": "^1.7.6", + "electron": "^1.8.8", "electron-packager": "^9.0.1" } } diff --git a/src/wallet/electron/wallet_rpc/rpc_wallet.py b/src/wallet/electron/rpc/rpc_wallet.py similarity index 72% rename from src/wallet/electron/wallet_rpc/rpc_wallet.py rename to src/wallet/electron/rpc/rpc_wallet.py index b104b5e2..56f3a447 100644 --- a/src/wallet/electron/wallet_rpc/rpc_wallet.py +++ b/src/wallet/electron/rpc/rpc_wallet.py @@ -5,14 +5,12 @@ import json from typing import Any from aiohttp import web -from blspy import ExtendedPrivateKey -from setproctitle import setproctitle from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.types.peer_info import PeerInfo -from src.util.config import load_config_cli, load_config -from src.wallet.wallet import Wallet +from src.util.config import load_config +from src.wallet.wallet_node import WalletNode class EnhancedJSONEncoder(json.JSONEncoder): @@ -44,14 +42,14 @@ class RpcWalletApiHandler: to the full node. """ - def __init__(self, wallet: Wallet): - self.wallet = wallet + def __init__(self, wallet_node: WalletNode): + self.wallet_node = wallet_node async def get_next_puzzle_hash(self, request) -> web.Response: """ Returns a new puzzlehash """ - puzzlehash = self.wallet.get_new_puzzlehash().hex() + puzzlehash = self.wallet_node.wallet.get_new_puzzlehash().hex() response = { "puzzlehash": puzzlehash, } @@ -62,47 +60,46 @@ class RpcWalletApiHandler: if "amount" in request_data and "puzzlehash" in request_data: amount = int(request_data["amount"]) puzzlehash = request_data["puzzlehash"] - tx = await self.wallet.generate_signed_transaction(amount, puzzlehash) + tx = await self.wallet_node.wallet.generate_signed_transaction( + amount, puzzlehash + ) if tx is None: - response = { - "success": False - } + response = {"success": False} return obj_to_response(response) - await self.wallet.push_transaction(tx) + await self.wallet_node.wallet.push_transaction(tx) - response = { - "success": True - } + response = {"success": True} return obj_to_response(response) - response = { - "success": False - } + response = {"success": False} return obj_to_response(response) async def get_server_ready(self, request) -> web.Response: - response = { - "success": True - } + response = {"success": True} return obj_to_response(response) async def get_transactions(self, request) -> web.Response: + transactions = ( + await self.wallet_node.wallet_state_manager.get_all_transactions() + ) - response = { - "success": True - } + response = {"success": True, "txs": transactions} return obj_to_response(response) async def get_wallet_balance(self, request) -> web.Response: + balance = await self.wallet_node.wallet.get_confirmed_balance() + pending_balance = await self.wallet_node.wallet.get_unconfirmed_balance() + response = { "success": True, - "confirmed_wallet_balance": 0, - "unconfirmed_wallet_balance": 0, + "confirmed_wallet_balance": balance, + "unconfirmed_wallet_balance": pending_balance, } + return obj_to_response(response) @@ -118,19 +115,19 @@ async def start_rpc_server(): raise RuntimeError( "Keys not generated. Run python3 ./scripts/regenerate_keys.py." ) - wallet = await Wallet.create(config, key_config) + wallet_node = await WalletNode.create(config, key_config) - server = ChiaServer(9257, wallet, NodeType.WALLET) - wallet.set_server(server) + server = ChiaServer(9257, wallet_node, NodeType.WALLET) + wallet_node.set_server(server) full_node_peer = PeerInfo( config["full_node_peer"]["host"], config["full_node_peer"]["port"] ) - _ = await server.start_server("127.0.0.1", wallet._on_connect) + _ = await server.start_server("127.0.0.1", wallet_node._on_connect) await asyncio.sleep(1) _ = await server.start_client(full_node_peer, None) - handler = RpcWalletApiHandler(wallet) + handler = RpcWalletApiHandler(wallet_node) app = web.Application() app.add_routes( [ @@ -155,8 +152,9 @@ async def start_rpc_server(): async def main(): cleanup = await start_rpc_server() - print('start running on {}') + print("start running on {}") await cleanup() -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/src/wallet/electron/wallet_rpc/.gitignore b/src/wallet/electron/wallet_rpc/.gitignore deleted file mode 100644 index d646835b..00000000 --- a/src/wallet/electron/wallet_rpc/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pyc -__pycache__/ diff --git a/src/wallet/electron/wallet_rpc/requirements.txt b/src/wallet/electron/wallet_rpc/requirements.txt deleted file mode 100644 index 37565d67..00000000 --- a/src/wallet/electron/wallet_rpc/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -zerorpc -pyzmq -future -msgpack-python -gevent -pyinstaller -pypiwin32 diff --git a/src/wallet/puzzles/load_clvm.py b/src/wallet/puzzles/load_clvm.py index 468f8668..ad046376 100644 --- a/src/wallet/puzzles/load_clvm.py +++ b/src/wallet/puzzles/load_clvm.py @@ -1,9 +1,11 @@ import pkg_resources -from src.types.hashable.Program import Program +from src.types.hashable.program import Program def load_clvm(filename): - clvm_hex = pkg_resources.resource_string(__name__, "%s.hex" % filename).decode("utf8") + clvm_hex = pkg_resources.resource_string(__name__, "%s.hex" % filename).decode( + "utf8" + ) clvm_blob = bytes.fromhex(clvm_hex) return Program.from_bytes(clvm_blob) diff --git a/src/wallet/puzzles/p2_conditions.py b/src/wallet/puzzles/p2_conditions.py index e5b839b2..c7f5431d 100644 --- a/src/wallet/puzzles/p2_conditions.py +++ b/src/wallet/puzzles/p2_conditions.py @@ -10,11 +10,9 @@ require a delegated puzzle program, so in those cases, this is just what the doctor ordered. """ -import clvm - from clvm_tools import binutils -from src.types.hashable.Program import Program +from src.types.hashable.program import Program # contract: diff --git a/src/wallet/puzzles/p2_delegated_conditions.py b/src/wallet/puzzles/p2_delegated_conditions.py index 2d5ca2a9..9cddad27 100644 --- a/src/wallet/puzzles/p2_delegated_conditions.py +++ b/src/wallet/puzzles/p2_delegated_conditions.py @@ -10,12 +10,10 @@ the doctor ordered. """ -import clvm - from clvm_tools import binutils from src.types.condition_opcodes import ConditionOpcode -from src.types.hashable.Program import Program +from src.types.hashable.program import Program def puzzle_for_pk(public_key): diff --git a/src/wallet/puzzles/p2_delegated_puzzle.py b/src/wallet/puzzles/p2_delegated_puzzle.py index a0f4d7b9..c81a73e1 100644 --- a/src/wallet/puzzles/p2_delegated_puzzle.py +++ b/src/wallet/puzzles/p2_delegated_puzzle.py @@ -17,15 +17,17 @@ from typing import List from clvm_tools import binutils from src.types.condition_opcodes import ConditionOpcode -from src.types.hashable.Program import Program +from src.types.hashable.program import Program from . import p2_conditions def puzzle_for_pk(public_key) -> Program: aggsig = ConditionOpcode.AGG_SIG[0] - TEMPLATE = (f"(c (c (q {aggsig}) (c (q 0x%s) (c (sha256tree (f (a))) (q ())))) " - f"((c (f (a)) (f (r (a))))))") + TEMPLATE = ( + f"(c (c (q {aggsig}) (c (q 0x%s) (c (sha256tree (f (a))) (q ())))) " + f"((c (f (a)) (f (r (a))))))" + ) return Program.to(binutils.assemble(TEMPLATE % public_key.hex())) diff --git a/src/wallet/puzzles/p2_delegated_puzzle_or_hidden_puzzle.py b/src/wallet/puzzles/p2_delegated_puzzle_or_hidden_puzzle.py index e5f2c029..a5bb267d 100644 --- a/src/wallet/puzzles/p2_delegated_puzzle_or_hidden_puzzle.py +++ b/src/wallet/puzzles/p2_delegated_puzzle_or_hidden_puzzle.py @@ -17,7 +17,7 @@ import hashlib from clvm_tools import binutils -from src.types.hashable.Program import Program +from src.types.hashable.program import Program from clvm import run_program from .load_clvm import load_clvm diff --git a/src/wallet/puzzles/p2_m_of_n_delegate_direct.py b/src/wallet/puzzles/p2_m_of_n_delegate_direct.py index 5d0967d0..f9b70770 100644 --- a/src/wallet/puzzles/p2_m_of_n_delegate_direct.py +++ b/src/wallet/puzzles/p2_m_of_n_delegate_direct.py @@ -5,7 +5,7 @@ This puzzle program is like p2_delegated_puzzle except instead of one public key it includes N public keys, any M of which needs to sign the delegated puzzle. """ -from src.types.hashable.Program import Program +from src.types.hashable.program import Program from clvm_tools import binutils from .load_clvm import load_clvm @@ -15,10 +15,11 @@ puzzle_prog_template = load_clvm("make_puzzle_m_of_n_direct.clvm") def puzzle_for_m_of_public_key_list(m, public_key_list): - format_tuple = tuple( + format_tuple = tuple([ binutils.disassemble(Program.to(_)) - for _ in (puzzle_prog_template, m, public_key_list)) - puzzle_src = "((c (q %s) (c (q %s) (c (q %s) (a)))))" % format_tuple + for _ in (puzzle_prog_template, m, public_key_list) + ]) + puzzle_src = "((c (q %s) (c (q %s) (c (q %s) (a)))))" % (format_tuple[0], format_tuple[1], format_tuple[2]) puzzle_prog = binutils.assemble(puzzle_src) return Program.to(puzzle_prog) diff --git a/src/wallet/puzzles/p2_puzzle_hash.py b/src/wallet/puzzles/p2_puzzle_hash.py index 49050347..27b16127 100644 --- a/src/wallet/puzzles/p2_puzzle_hash.py +++ b/src/wallet/puzzles/p2_puzzle_hash.py @@ -5,11 +5,9 @@ In this puzzle program, the solution must be a reveal of the puzzle with the giv hash along with its solution. """ -import clvm - from clvm_tools import binutils -from src.types.hashable.Program import Program +from src.types.hashable.program import Program """ solution: (puzzle_reveal . solution_to_puzzle) diff --git a/src/wallet/puzzles/puzzle_utils.py b/src/wallet/puzzles/puzzle_utils.py index 7a526b7e..9276f182 100644 --- a/src/wallet/puzzles/puzzle_utils.py +++ b/src/wallet/puzzles/puzzle_utils.py @@ -1,7 +1,6 @@ from src.util.condition_tools import ConditionOpcode - def make_create_coin_condition(puzzle_hash, amount): return [ConditionOpcode.CREATE_COIN, puzzle_hash, amount] @@ -17,6 +16,7 @@ def make_assert_coin_consumed_condition(coin_name): def make_assert_my_coin_id_condition(coin_name): return [ConditionOpcode.ASSERT_MY_COIN_ID, coin_name] + def make_assert_block_index_exceeds_condition(block_index): return [ConditionOpcode.ASSERT_BLOCK_INDEX_EXCEEDS, block_index] @@ -26,4 +26,4 @@ def make_assert_block_age_exceeds_condition(block_index): def make_assert_time_exceeds_condition(time): - return [ConditionOpcode.ASSERT_TIME_EXCEEDS, time] \ No newline at end of file + return [ConditionOpcode.ASSERT_TIME_EXCEEDS, time] diff --git a/src/wallet/transaction_record.py b/src/wallet/transaction_record.py new file mode 100644 index 00000000..574d15b5 --- /dev/null +++ b/src/wallet/transaction_record.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from typing import Optional, List + +from src.types.hashable.coin import Coin +from src.types.hashable.spend_bundle import SpendBundle +from src.types.sized_bytes import bytes32 +from src.util.streamable import Streamable, streamable +from src.util.ints import uint32, uint64 + + +@dataclass(frozen=True) +@streamable +class TransactionRecord(Streamable): + """ + Used for storing transaction data and status in wallets + """ + + confirmed_block_index: uint32 + created_at_index: uint32 + confirmed: bool + sent: bool + created_at_time: uint64 + spend_bundle: Optional[SpendBundle] + additions: List[Coin] + removals: List[Coin] + + def name(self) -> bytes32: + if self.spend_bundle: + return self.spend_bundle.name() + return self.get_hash() diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index b3d3c850..8b98539d 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -1,32 +1,21 @@ -from pathlib import Path -from typing import Dict, Optional, List, Set, Tuple +from typing import Dict, Optional, List, Tuple import clvm from blspy import ExtendedPrivateKey, PublicKey import logging -import src.protocols.wallet_protocol -from src.full_node import OutboundMessageGenerator -from src.protocols.wallet_protocol import ProofHash -from src.protocols.full_node_protocol import RespondTransaction from src.server.outbound_message import OutboundMessage, NodeType, Message, Delivery from src.server.server import ChiaServer -from src.types.full_block import additions_for_npc +from src.protocols import full_node_protocol from src.types.hashable.BLSSignature import BLSSignature -from src.types.hashable.Coin import Coin -from src.types.hashable.CoinRecord import CoinRecord -from src.types.hashable.CoinSolution import CoinSolution -from src.types.hashable.Program import Program -from src.types.hashable.SpendBundle import SpendBundle -from src.types.name_puzzle_condition import NPC +from src.types.hashable.coin_solution import CoinSolution +from src.types.hashable.program import Program +from src.types.hashable.spend_bundle import SpendBundle from src.types.sized_bytes import bytes32 -from src.util.hash import std_hash -from src.util.api_decorators import api_request from src.util.condition_tools import ( conditions_for_solution, conditions_by_opcode, hash_key_pairs_for_conditions_dict, ) -from src.util.ints import uint32, uint64 -from src.util.mempool_check_conditions import get_name_puzzle_conditions +from src.util.ints import uint64 from src.wallet.BLSPrivateKey import BLSPrivateKey from src.wallet.puzzles.p2_conditions import puzzle_for_conditions from src.wallet.puzzles.p2_delegated_puzzle import puzzle_for_pk @@ -36,7 +25,8 @@ from src.wallet.puzzles.puzzle_utils import ( make_assert_coin_consumed_condition, make_create_coin_condition, ) -from src.wallet.wallet_store import WalletStore + +from src.wallet.wallet_state_manager import WalletStateManager class Wallet: @@ -46,22 +36,8 @@ class Wallet: server: Optional[ChiaServer] next_address: int = 0 pubkey_num_lookup: Dict[bytes, int] - tmp_coins: Set[Coin] - wallet_store: WalletStore - header_hash: List[bytes32] - start_index: int + wallet_state_manager: WalletStateManager - unconfirmed_removals: Set[Coin] - unconfirmed_removal_amount: int - - unconfirmed_additions: Set[Coin] - unconfirmed_addition_amount: int - - # This dict maps coin_id to SpendBundle, it will contain duplicate values by design - coin_spend_bundle_map: Dict[bytes32, SpendBundle] - - # Spendbundle_ID : Spendbundle - pending_spend_bundles: Dict[bytes32, SpendBundle] log: logging.Logger # TODO Don't allow user to send tx until wallet is synced @@ -71,9 +47,13 @@ class Wallet: send_queue: Dict[bytes32, SpendBundle] @staticmethod - async def create(config: Dict, key_config: Dict, name: str = None): + async def create( + config: Dict, + key_config: Dict, + wallet_state_manager: WalletStateManager, + name: str = None, + ): self = Wallet() - print("init wallet") self.config = config self.key_config = key_config sk_hex = self.key_config["wallet_sk"] @@ -83,22 +63,9 @@ class Wallet: else: self.log = logging.getLogger(__name__) + self.wallet_state_manager = wallet_state_manager self.pubkey_num_lookup = {} - self.tmp_coins = set() - pub_hex = self.private_key.get_public_key().serialize().hex() - path = Path(f"wallet_db_{pub_hex}.db") - self.wallet_store = await WalletStore.create(path) - self.header_hash = [] - self.unconfirmed_additions = set() - self.unconfirmed_removals = set() - self.pending_spend_bundles = {} - self.coin_spend_bundle_map = {} - self.unconfirmed_addition_amount = 0 - self.unconfirmed_removal_amount = 0 - self.synced = False - - self.send_queue = {} self.server = None return self @@ -107,23 +74,14 @@ class Wallet: pubkey = self.private_key.public_child(self.next_address).get_public_key() self.pubkey_num_lookup[pubkey.serialize()] = self.next_address self.next_address = self.next_address + 1 + self.wallet_state_manager.next_address = self.next_address return pubkey async def get_confirmed_balance(self) -> uint64: - record_list: Set[ - CoinRecord - ] = await self.wallet_store.get_coin_records_by_spent(False) - amount: uint64 = uint64(0) - - for record in record_list: - amount = uint64(amount + record.coin.amount) - - return uint64(amount) + return await self.wallet_state_manager.get_confirmed_balance() async def get_unconfirmed_balance(self) -> uint64: - confirmed = await self.get_confirmed_balance() - result = confirmed - self.unconfirmed_removal_amount + self.unconfirmed_addition_amount - return uint64(result) + return await self.wallet_state_manager.get_unconfirmed_balance() def can_generate_puzzle_hash(self, hash: bytes32) -> bool: return any( @@ -147,50 +105,9 @@ class Wallet: def get_new_puzzlehash(self) -> bytes32: puzzle: Program = self.get_new_puzzle() puzzlehash: bytes32 = puzzle.get_hash() + self.wallet_state_manager.puzzlehash_set.add(puzzlehash) return puzzlehash - async def select_coins(self, amount) -> Optional[Set[Coin]]: - - if amount > await self.get_unconfirmed_balance(): - return None - - unspent: Set[ - CoinRecord - ] = await self.wallet_store.get_coin_records_by_spent(False) - sum = 0 - used_coins: Set = set() - - """ - Try to use coins from the store, if there isn't enough of "unused" - coins use change coins that are not confirmed yet - """ - for coinrecord in unspent: - if sum >= amount: - break - if coinrecord.coin.name in self.unconfirmed_removals: - continue - sum += coinrecord.coin.amount - used_coins.add(coinrecord.coin) - - """ - This happens when we couldn't use one of the coins because it's already used - but unconfirmed, and we are waiting for the change. (unconfirmed_additions) - """ - if sum < amount: - for coin in self.unconfirmed_additions: - if sum > amount: - break - if coin.name in self.unconfirmed_removals: - continue - sum += coin.amount - used_coins.add(coin) - - if sum >= amount: - return used_coins - else: - # This shouldn't happen because of: if amount > self.get_unconfirmed_balance(): - return None - def set_server(self, server: ChiaServer): self.server = server @@ -227,7 +144,7 @@ class Wallet: async def generate_unsigned_transaction( self, amount: int, newpuzzlehash: bytes32, fee: int = 0 ) -> List[Tuple[Program, CoinSolution]]: - utxos = await self.select_coins(amount + fee) + utxos = await self.wallet_state_manager.select_coins(amount + fee) if utxos is None: return [] spends: List[Tuple[Program, CoinSolution]] = [] @@ -246,8 +163,7 @@ class Wallet: if change > 0: changepuzzlehash = self.get_new_puzzlehash() primaries.append({"puzzlehash": changepuzzlehash, "amount": change}) - # add change coin into temp_utxo set - self.tmp_coins.add(Coin(coin.name(), changepuzzlehash, uint64(change))) + solution = self.make_solution(primaries=primaries) output_created = True else: @@ -286,6 +202,7 @@ class Wallet: async def generate_signed_transaction( self, amount, newpuzzlehash, fee: int = 0 ) -> Optional[SpendBundle]: + """ Use this to generate transaction. """ transaction = await self.generate_unsigned_transaction( amount, newpuzzlehash, fee ) @@ -293,146 +210,17 @@ class Wallet: return None return self.sign_transaction(transaction) - async def coin_removed(self, coin_name: bytes32, index: uint32): - self.log.info("remove coin") - await self.wallet_store.set_spent(coin_name, index) - - async def coin_added(self, coin: Coin, index: uint32, coinbase: bool): - self.log.info("add coin") - coin_record: CoinRecord = CoinRecord(coin, index, uint32(0), False, coinbase) - await self.wallet_store.add_coin_record(coin_record) - - async def _on_connect(self) -> OutboundMessageGenerator: - """ - Whenever we connect to a FullNode we request new proof_hashes by sending last proof hash we have - """ - self.log.info(f"Requesting proof hashes") - request = ProofHash(std_hash(b"deadbeef")) - yield OutboundMessage( - NodeType.FULL_NODE, - Message("request_proof_hashes", request), - Delivery.BROADCAST, - ) - - @api_request - async def proof_hash( - self, request: src.protocols.wallet_protocol.ProofHash - ) -> OutboundMessageGenerator: - """ - Received a proof hash from the FullNode - """ - self.log.info(f"Received a new proof hash: {request}") - reply_request = ProofHash(std_hash(b"a")) - # TODO Store and decide if we want full proof for this proof hash - yield OutboundMessage( - NodeType.FULL_NODE, - Message("request_full_proof_for_hash", reply_request), - Delivery.RESPOND, - ) - - @api_request - async def full_proof_for_hash( - self, request: src.protocols.wallet_protocol.FullProofForHash - ): - """ - We've received a full proof for hash we requested - """ - # TODO Validate full proof - self.log.info(f"Received new proof: {request}") - - @api_request - async def received_body(self, response: src.protocols.wallet_protocol.RespondBody): - """ - Called when body is received from the FullNode - """ - - # Retry sending queued up transactions - await self.retry_send_queue() - - additions: List[Coin] = [] - - if self.can_generate_puzzle_hash(response.body.coinbase.puzzle_hash): - await self.coin_added(response.body.coinbase, response.height, True) - if self.can_generate_puzzle_hash(response.body.fees_coin.puzzle_hash): - await self.coin_added(response.body.fees_coin, response.height, True) - - npc_list: List[NPC] - if response.body.transactions: - error, npc_list, cost = await get_name_puzzle_conditions( - response.body.transactions - ) - - additions.extend(additions_for_npc(npc_list)) - - for added_coin in additions: - if self.can_generate_puzzle_hash(added_coin.puzzle_hash): - await self.coin_added(added_coin, response.height, False) - - for npc in npc_list: - if self.can_generate_puzzle_hash(npc.puzzle_hash): - await self.coin_removed(npc.coin_name, response.height) - - @api_request - async def new_tip(self, header: src.protocols.wallet_protocol.Header): - self.log.info("new tip received") - - async def retry_send_queue(self): - for key, val in self.send_queue: - await self._send_transaction(val) - - def remove_from_queue(self, spendbundle_id: bytes32): - if spendbundle_id in self.send_queue: - del self.send_queue[spendbundle_id] - async def push_transaction(self, spend_bundle: SpendBundle): - """ Use this API to make transactions. """ - self.send_queue[spend_bundle.name()] = spend_bundle - additions: List[Coin] = spend_bundle.additions() - removals: List[Coin] = spend_bundle.removals() - - addition_amount = 0 - for coin in additions: - if self.can_generate_puzzle_hash(coin.puzzle_hash): - self.unconfirmed_additions.add(coin) - self.coin_spend_bundle_map[coin.name()] = spend_bundle - addition_amount += coin.amount - - removal_amount = 0 - for coin in removals: - self.unconfirmed_removals.add(coin) - self.coin_spend_bundle_map[coin.name()] = spend_bundle - removal_amount += coin.amount - - # Update unconfirmed state - self.unconfirmed_removal_amount += removal_amount - self.unconfirmed_addition_amount += addition_amount - - self.pending_spend_bundles[spend_bundle.name()] = spend_bundle + """ Use this API to send transactions. """ + await self.wallet_state_manager.add_pending_transaction(spend_bundle) await self._send_transaction(spend_bundle) async def _send_transaction(self, spend_bundle: SpendBundle): - """ Sends spendbundle to connected full Nodes.""" - - msg = OutboundMessage( - NodeType.FULL_NODE, - Message("respond_transaction", RespondTransaction(spend_bundle)), - Delivery.BROADCAST, - ) if self.server: + msg = OutboundMessage( + NodeType.FULL_NODE, + Message("respond_transaction", full_node_protocol.RespondTransaction(spend_bundle)), + Delivery.BROADCAST, + ) async for reply in self.server.push_message(msg): self.log.info(reply) - - @api_request - async def transaction_ack(self, ack: src.protocols.wallet_protocol.TransactionAck): - if ack.status: - self.remove_from_queue(ack.txid) - self.log.info(f"SpendBundle has been received by the FullNode. id: {id}") - else: - self.log.info(f"SpendBundle has been rejected by the FullNode. id: {id}") - - async def requestLCA(self): - msg = OutboundMessage( - NodeType.FULL_NODE, Message("request_lca", None), Delivery.BROADCAST, - ) - async for reply in self.server.push_message(msg): - self.log.info(reply) diff --git a/src/wallet/wallet_node.py b/src/wallet/wallet_node.py new file mode 100644 index 00000000..632dbde4 --- /dev/null +++ b/src/wallet/wallet_node.py @@ -0,0 +1,203 @@ +from pathlib import Path +from typing import Dict, Optional, List +from blspy import ExtendedPrivateKey +import logging +import src.protocols.wallet_protocol +from src.full_node.full_node import OutboundMessageGenerator +from src.protocols.wallet_protocol import ProofHash +from src.server.outbound_message import OutboundMessage, NodeType, Message, Delivery +from src.server.server import ChiaServer +from src.types.full_block import additions_for_npc +from src.types.hashable.coin import Coin +from src.types.hashable.spend_bundle import SpendBundle +from src.types.name_puzzle_condition import NPC +from src.types.sized_bytes import bytes32 +from src.util.hash import std_hash +from src.util.api_decorators import api_request +from src.util.ints import uint32 +from src.util.mempool_check_conditions import get_name_puzzle_conditions +from src.wallet.wallet import Wallet +from src.wallet.wallet_state_manager import WalletStateManager +from src.wallet.wallet_store import WalletStore +from src.wallet.wallet_transaction_store import WalletTransactionStore + + +class WalletNode: + private_key: ExtendedPrivateKey + key_config: Dict + config: Dict + server: Optional[ChiaServer] + wallet_store: WalletStore + wallet_state_manager: WalletStateManager + header_hash: List[bytes32] + start_index: int + log: logging.Logger + wallet: Wallet + tx_store: WalletTransactionStore + + @staticmethod + async def create(config: Dict, key_config: Dict, name: str = None): + self = WalletNode() + self.config = config + self.key_config = key_config + sk_hex = self.key_config["wallet_sk"] + self.private_key = ExtendedPrivateKey.from_bytes(bytes.fromhex(sk_hex)) + if name: + self.log = logging.getLogger(name) + else: + self.log = logging.getLogger(__name__) + + pub_hex = self.private_key.get_public_key().serialize().hex() + path = Path(f"wallet_db_{pub_hex}.db") + self.wallet_store = await WalletStore.create(path) + self.tx_store = await WalletTransactionStore.create(path) + + self.wallet_state_manager = await WalletStateManager.create( + config, key_config, self.wallet_store, self.tx_store + ) + self.wallet = await Wallet.create(config, key_config, self.wallet_state_manager) + + self.server = None + + return self + + def set_server(self, server: ChiaServer): + self.server = server + self.wallet.set_server(server) + + async def _on_connect(self) -> OutboundMessageGenerator: + """ + Whenever we connect to a FullNode we request new proof_hashes by sending last proof hash we have + """ + self.log.info(f"Requesting proof hashes") + request = ProofHash(std_hash(b"deadbeef")) + yield OutboundMessage( + NodeType.FULL_NODE, + Message("request_proof_hashes", request), + Delivery.BROADCAST, + ) + + @api_request + async def proof_hash( + self, request: src.protocols.wallet_protocol.ProofHash + ) -> OutboundMessageGenerator: + """ + Received a proof hash from the FullNode + """ + self.log.info(f"Received a new proof hash: {request}") + reply_request = ProofHash(std_hash(b"a")) + # TODO Store and decide if we want full proof for this proof hash + yield OutboundMessage( + NodeType.FULL_NODE, + Message("request_full_proof_for_hash", reply_request), + Delivery.RESPOND, + ) + + @api_request + async def full_proof_for_hash( + self, request: src.protocols.wallet_protocol.FullProofForHash + ): + """ + We've received a full proof for hash we requested + """ + # TODO Validate full proof + self.log.info(f"Received new proof: {request}") + + @api_request + async def received_body(self, response: src.protocols.wallet_protocol.RespondBody): + """ + Called when body is received from the FullNode + """ + + # Retry sending queued up transactions + await self.retry_send_queue() + + additions: List[Coin] = [] + + if self.wallet.can_generate_puzzle_hash(response.body.coinbase.puzzle_hash): + await self.wallet_state_manager.coin_added( + response.body.coinbase, response.height, True + ) + if self.wallet.can_generate_puzzle_hash(response.body.fees_coin.puzzle_hash): + await self.wallet_state_manager.coin_added( + response.body.fees_coin, response.height, True + ) + + npc_list: List[NPC] + if response.body.transactions: + error, npc_list, cost = get_name_puzzle_conditions( + response.body.transactions + ) + + additions.extend(additions_for_npc(npc_list)) + + for added_coin in additions: + if self.wallet.can_generate_puzzle_hash(added_coin.puzzle_hash): + await self.wallet_state_manager.coin_added( + added_coin, response.height, False + ) + + for npc in npc_list: + if self.wallet.can_generate_puzzle_hash(npc.puzzle_hash): + await self.wallet_state_manager.coin_removed( + npc.coin_name, response.height + ) + + @api_request + async def new_lca(self, header: src.protocols.wallet_protocol.Header): + self.log.info("new tip received") + + async def retry_send_queue(self): + records = await self.wallet_state_manager.get_send_queue() + for record in records: + if record.spend_bundle: + await self._send_transaction(record.spend_bundle) + + async def _send_transaction(self, spend_bundle: SpendBundle): + """ Sends spendbundle to connected full Nodes.""" + await self.wallet_state_manager.add_pending_transaction(spend_bundle) + + msg = OutboundMessage( + NodeType.FULL_NODE, + Message("wallet_transaction", spend_bundle), + Delivery.BROADCAST, + ) + if self.server: + async for reply in self.server.push_message(msg): + self.log.info(reply) + + async def _request_add_list(self, height: uint32, header_hash: bytes32): + obj = src.protocols.wallet_protocol.RequestAdditions(height, header_hash) + msg = OutboundMessage( + NodeType.FULL_NODE, Message("request_additions", obj), Delivery.BROADCAST, + ) + if self.server: + async for reply in self.server.push_message(msg): + self.log.info(reply) + + @api_request + async def response_additions( + self, response: src.protocols.wallet_protocol.Additions + ): + print(response) + + @api_request + async def response_additions_rejected( + self, response: src.protocols.wallet_protocol.RequestAdditions + ): + print(f"request rejected {response}") + + @api_request + async def transaction_ack(self, ack: src.protocols.wallet_protocol.TransactionAck): + if ack.status: + await self.wallet_state_manager.remove_from_queue(ack.txid) + self.log.info(f"SpendBundle has been received by the FullNode. id: {id}") + else: + self.log.info(f"SpendBundle has been rejected by the FullNode. id: {id}") + + async def requestLCA(self): + msg = OutboundMessage( + NodeType.FULL_NODE, Message("request_lca", None), Delivery.BROADCAST, + ) + async for reply in self.server.push_message(msg): + self.log.info(reply) diff --git a/src/wallet/wallet_state_manager.py b/src/wallet/wallet_state_manager.py new file mode 100644 index 00000000..376a451c --- /dev/null +++ b/src/wallet/wallet_state_manager.py @@ -0,0 +1,188 @@ +import time +from typing import Dict, Optional, List, Set +import logging +from src.types.hashable.coin import Coin +from src.types.hashable.coin_record import CoinRecord +from src.types.hashable.spend_bundle import SpendBundle +from src.types.sized_bytes import bytes32 +from src.util.ints import uint32, uint64 +from src.wallet.transaction_record import TransactionRecord +from src.wallet.wallet_store import WalletStore +from src.wallet.wallet_transaction_store import WalletTransactionStore + + +class WalletStateManager: + key_config: Dict + config: Dict + wallet_store: WalletStore + tx_store: WalletTransactionStore + header_hash: List[bytes32] + start_index: int + next_address: int + + log: logging.Logger + + # TODO Don't allow user to send tx until wallet is synced + synced: bool + puzzlehash_set: set + + @staticmethod + async def create( + config: Dict, + key_config: Dict, + wallet_store: WalletStore, + tx_store: WalletTransactionStore, + name: str = None, + ): + self = WalletStateManager() + self.config = config + + if name: + self.log = logging.getLogger(name) + else: + self.log = logging.getLogger(__name__) + + self.header_hash = [] + self.wallet_store = wallet_store + self.tx_store = tx_store + self.synced = False + self.next_address = 0 + + self.puzzlehash_set = set() + return self + + async def get_confirmed_balance(self) -> uint64: + record_list: Set[ + CoinRecord + ] = await self.wallet_store.get_coin_records_by_spent(False) + amount: uint64 = uint64(0) + + for record in record_list: + amount = uint64(amount + record.coin.amount) + + return uint64(amount) + + async def get_unconfirmed_balance(self) -> uint64: + confirmed = await self.get_confirmed_balance() + unconfirmed_tx = await self.tx_store.get_not_confirmed() + addition_amount = 0 + removal_amount = 0 + + for record in unconfirmed_tx: + for coin in record.additions: + if coin.puzzle_hash in self.puzzlehash_set: + addition_amount += coin.amount + for coin in record.removals: + removal_amount += coin.amount + + result = confirmed - removal_amount + addition_amount + return uint64(result) + + async def unconfirmed_additions(self) -> Dict[bytes32, Coin]: + additions: Dict[bytes32, Coin] = {} + unconfirmed_tx = await self.tx_store.get_not_confirmed() + for record in unconfirmed_tx: + for coin in record.additions: + additions[coin.name()] = coin + return additions + + async def unconfirmed_removals(self) -> Dict[bytes32, Coin]: + removals: Dict[bytes32, Coin] = {} + unconfirmed_tx = await self.tx_store.get_not_confirmed() + for record in unconfirmed_tx: + for coin in record.removals: + removals[coin.name()] = coin + return removals + + async def select_coins(self, amount) -> Optional[Set[Coin]]: + + if amount > await self.get_unconfirmed_balance(): + return None + + unspent: Set[CoinRecord] = await self.wallet_store.get_coin_records_by_spent( + False + ) + sum = 0 + used_coins: Set = set() + + """ + Try to use coins from the store, if there isn't enough of "unused" + coins use change coins that are not confirmed yet + """ + for coinrecord in unspent: + if sum >= amount: + break + if coinrecord.coin.name in await self.unconfirmed_removals(): + continue + sum += coinrecord.coin.amount + used_coins.add(coinrecord.coin) + + """ + This happens when we couldn't use one of the coins because it's already used + but unconfirmed, and we are waiting for the change. (unconfirmed_additions) + """ + if sum < amount: + for coin in (await self.unconfirmed_additions()).values(): + if sum > amount: + break + if coin.name in (await self.unconfirmed_removals()).values(): + continue + sum += coin.amount + used_coins.add(coin) + + if sum >= amount: + return used_coins + else: + # This shouldn't happen because of: if amount > self.get_unconfirmed_balance(): + return None + + async def coin_removed(self, coin_name: bytes32, index: uint32): + """ + Called when coin gets spent + """ + await self.wallet_store.set_spent(coin_name, index) + + async def coin_added(self, coin: Coin, index: uint32, coinbase: bool): + """ + Adding coin to the db + """ + coin_record: CoinRecord = CoinRecord(coin, index, uint32(0), False, coinbase) + await self.wallet_store.add_coin_record(coin_record) + + async def add_pending_transaction(self, spend_bundle: SpendBundle): + """ + Called from wallet_node before new transaction is sent to the full_node + """ + now = uint64(int(time.time())) + add_list: List[Coin] = [] + rem_list: List[Coin] = [] + for add in spend_bundle.additions(): + add_list.append(add) + for rem in spend_bundle.removals(): + rem_list.append(rem) + + # Wallet node will use this queue to retry sending this transaction until full nodes receives it + tx_record = TransactionRecord( + uint32(0), uint32(0), False, False, now, spend_bundle, add_list, rem_list + ) + await self.tx_store.add_transaction_record(tx_record) + + async def remove_from_queue(self, spendbundle_id: bytes32): + """ + Full node received our transaction, no need to keep it in queue anymore + """ + await self.tx_store.set_sent(spendbundle_id) + + async def get_send_queue(self) -> List[TransactionRecord]: + """ + Wallet Node uses this to retry sending transactions + """ + records = await self.tx_store.get_not_sent() + return records + + async def get_all_transactions(self) -> List[TransactionRecord]: + """ + Retrieves all confirmed and pending transactions + """ + records = await self.tx_store.get_all_transactions() + return records diff --git a/src/wallet/wallet_store.py b/src/wallet/wallet_store.py index 3a335430..0e6a386f 100644 --- a/src/wallet/wallet_store.py +++ b/src/wallet/wallet_store.py @@ -2,8 +2,8 @@ import asyncio from typing import Dict, Optional, List, Set from pathlib import Path import aiosqlite -from src.types.hashable.Coin import Coin -from src.types.hashable.CoinRecord import CoinRecord +from src.types.hashable.coin import Coin +from src.types.hashable.coin_record import CoinRecord from src.types.sized_bytes import bytes32 from src.util.ints import uint32 @@ -17,7 +17,7 @@ class WalletStore: # Whether or not we are syncing sync_mode: bool = False lock: asyncio.Lock - lca_coin_records: Dict[str, CoinRecord] + coin_record_cache: Dict[str, CoinRecord] cache_size: uint32 @classmethod @@ -61,7 +61,7 @@ class WalletStore: await self.coin_record_db.commit() # Lock self.lock = asyncio.Lock() # external - self.lca_coin_records = dict() + self.coin_record_cache = dict() return self async def close(self): @@ -89,11 +89,11 @@ class WalletStore: ) await cursor.close() await self.coin_record_db.commit() - self.lca_coin_records[record.coin.name().hex()] = record - if len(self.lca_coin_records) > self.cache_size: - while len(self.lca_coin_records) > self.cache_size: - first_in = list(self.lca_coin_records.keys())[0] - del self.lca_coin_records[first_in] + self.coin_record_cache[record.coin.name().hex()] = record + if len(self.coin_record_cache) > self.cache_size: + while len(self.coin_record_cache) > self.cache_size: + first_in = list(self.coin_record_cache.keys())[0] + del self.coin_record_cache[first_in] # Update coin_record to be spent in DB async def set_spent(self, coin_name: bytes32, index: uint32): @@ -102,13 +102,13 @@ class WalletStore: return spent: CoinRecord = CoinRecord( current.coin, current.confirmed_block_index, index, True, current.coinbase, - ) # type: ignore # noqa + ) await self.add_coin_record(spent) # Checks DB and DiffStores for CoinRecord with coin_name and returns it async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: - if coin_name.hex() in self.lca_coin_records: - return self.lca_coin_records[coin_name.hex()] + if coin_name.hex() in self.coin_record_cache: + return self.coin_record_cache[coin_name.hex()] cursor = await self.coin_record_db.execute( "SELECT * from coin_record WHERE coin_name=?", (coin_name.hex(),) ) @@ -157,7 +157,7 @@ class WalletStore: async def rollback_lca_to_block(self, block_index): # Update memory cache delete_queue: bytes32 = [] - for coin_name, coin_record in self.lca_coin_records.items(): + for coin_name, coin_record in self.coin_record_cache.items(): if coin_record.spent_block_index > block_index: new_record = CoinRecord( coin_record.coin, @@ -166,12 +166,12 @@ class WalletStore: False, coin_record.coinbase, ) - self.lca_coin_records[coin_record.coin.name().hex()] = new_record + self.coin_record_cache[coin_record.coin.name().hex()] = new_record if coin_record.confirmed_block_index > block_index: delete_queue.append(coin_name) for coin_name in delete_queue: - del self.lca_coin_records[coin_name] + del self.coin_record_cache[coin_name] # Delete from storage c1 = await self.coin_record_db.execute( diff --git a/src/wallet/wallet_transaction_store.py b/src/wallet/wallet_transaction_store.py new file mode 100644 index 00000000..24883b11 --- /dev/null +++ b/src/wallet/wallet_transaction_store.py @@ -0,0 +1,185 @@ +import asyncio +from typing import Dict, Optional, List +from pathlib import Path +import aiosqlite +from src.types.sized_bytes import bytes32 +from src.util.ints import uint32 +from src.wallet.transaction_record import TransactionRecord + + +class WalletTransactionStore: + """ + This object handles CoinRecords in DB used by wallet. + """ + + transaction_db: aiosqlite.Connection + # Whether or not we are syncing + sync_mode: bool = False + lock: asyncio.Lock + cache_size: uint32 + tx_record_cache: Dict[bytes32, TransactionRecord] + + @classmethod + async def create(cls, db_path: Path, cache_size: uint32 = uint32(600000)): + self = cls() + + self.cache_size = cache_size + + self.transaction_db = await aiosqlite.connect(db_path) + await self.transaction_db.execute( + ( + f"CREATE TABLE IF NOT EXISTS transaction_record(" + f"bundle_id text PRIMARY KEY," + f" confirmed_index bigint," + f" created_at_index bigint," + f" confirmed int," + f" sent int," + f" created_at_time bigint," + f" transaction_record blob)" + ) + ) + + # Useful for reorg lookups + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_confirmed_index on transaction_record(confirmed_index)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_created_index on transaction_record(created_at_index)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_confirmed on transaction_record(confirmed)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_sent on transaction_record(sent)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_created_time on transaction_record(created_at_time)" + ) + + await self.transaction_db.commit() + # Lock + self.lock = asyncio.Lock() # external + self.tx_record_cache = dict() + return self + + async def close(self): + await self.transaction_db.close() + + async def _init_cache(self): + print("init cache here") + + async def _clear_database(self): + cursor = await self.transaction_db.execute("DELETE FROM transaction_record") + await cursor.close() + await self.transaction_db.commit() + + # Store TransactionRecord in DB and Cache + async def add_transaction_record(self, record: TransactionRecord) -> None: + cursor = await self.transaction_db.execute( + "INSERT OR REPLACE INTO transaction_record VALUES(?, ?, ?, ?, ?, ?, ?)", + ( + record.name().hex(), + record.confirmed_block_index, + record.created_at_index, + int(record.confirmed), + int(record.sent), + record.created_at_time, + bytes(record), + ), + ) + await cursor.close() + await self.transaction_db.commit() + self.tx_record_cache[record.name().hex()] = record + if len(self.tx_record_cache) > self.cache_size: + while len(self.tx_record_cache) > self.cache_size: + first_in = list(self.tx_record_cache.keys())[0] + self.tx_record_cache.pop(first_in) + + # Update transaction_record to be confirmed in DB + async def set_confirmed(self, id: bytes32, index: uint32): + current: Optional[TransactionRecord] = await self.get_transaction_record(id) + if current is None: + return + tx: TransactionRecord = TransactionRecord( + index, + current.created_at_index, + True, + current.sent, + current.created_at_time, + current.spend_bundle, + current.additions, + current.removals, + ) + await self.add_transaction_record(tx) + + # Update transaction_record to be sent in DB + async def set_sent(self, id: bytes32): + current: Optional[TransactionRecord] = await self.get_transaction_record(id) + if current is None: + return + tx: TransactionRecord = TransactionRecord( + current.confirmed_block_index, + current.created_at_index, + current.confirmed, + True, + current.created_at_time, + current.spend_bundle, + current.additions, + current.removals, + ) + await self.add_transaction_record(tx) + + # Checks DB and cache for TransactionRecord with id: id and returns it + async def get_transaction_record(self, id: bytes32) -> Optional[TransactionRecord]: + if id.hex() in self.tx_record_cache: + return self.tx_record_cache[id.hex()] + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record WHERE bundle_id=?", (id.hex(),) + ) + row = await cursor.fetchone() + await cursor.close() + if row is not None: + record = TransactionRecord.from_bytes(row[6]) + return record + return None + + async def get_not_sent(self) -> List[TransactionRecord]: + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record WHERE sent=?", (0,) + ) + rows = await cursor.fetchall() + await cursor.close() + records = [] + for row in rows: + record = TransactionRecord.from_bytes(row[6]) + records.append(record) + + return records + + async def get_not_confirmed(self) -> List[TransactionRecord]: + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record WHERE confirmed=?", (0,) + ) + rows = await cursor.fetchall() + await cursor.close() + records = [] + for row in rows: + record = TransactionRecord.from_bytes(row[6]) + records.append(record) + + return records + + async def get_all_transactions(self) -> List[TransactionRecord]: + cursor = await self.transaction_db.execute("SELECT * from transaction_record") + rows = await cursor.fetchall() + await cursor.close() + records = [] + for row in rows: + record = TransactionRecord.from_bytes(row[6]) + records.append(record) + + return records diff --git a/tests/block_tools.py b/tests/block_tools.py index d54c084d..5e711a02 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -6,6 +6,7 @@ from pathlib import Path import blspy from blspy import PrependSignature, PrivateKey, PublicKey +from chiabip158 import PyBIP158 from chiapos import DiskPlotter, DiskProver from lib.chiavdf.inkfish.classgroup import ClassGroup @@ -18,19 +19,22 @@ from src.pool import create_coinbase_coin_and_signature from src.types.body import Body from src.types.challenge import Challenge from src.types.classgroup import ClassgroupElement -from src.types.full_block import FullBlock +from src.types.full_block import FullBlock, additions_for_npc from src.types.hashable.BLSSignature import BLSSignature -from src.types.hashable.Coin import Coin -from src.types.hashable.Program import Program +from src.types.hashable.coin import Coin, hash_coin_list +from src.types.hashable.program import Program from src.types.header import Header, HeaderData from src.types.proof_of_space import ProofOfSpace from src.types.proof_of_time import ProofOfTime from src.types.sized_bytes import bytes32 +from src.util.merkle_set import MerkleSet from src.util.errors import NoProofsOfSpaceFound from src.util.ints import uint8, uint32, uint64 from src.util.hash import std_hash # Can't go much lower than 19, since plots start having no solutions +from src.util.mempool_check_conditions import get_name_puzzle_conditions + k: uint8 = uint8(19) # Uses many plots for testing, in order to guarantee proofs of space at every height num_plots = 40 @@ -444,17 +448,63 @@ class BlockTools: extension_data, ) + # Create filter + byte_array_tx: List[bytes32] = [] + tx_additions: List[Coin] = [] + tx_removals: List[bytes32] = [] + if transactions: + error, npc_list, _ = get_name_puzzle_conditions(transactions) + additions: List[Coin] = additions_for_npc(npc_list) + for coin in additions: + tx_additions.append(coin) + byte_array_tx.append(bytearray(coin.puzzle_hash)) + for npc in npc_list: + tx_removals.append(npc.coin_name) + byte_array_tx.append(bytearray(npc.coin_name)) + + byte_array_tx.append(bytearray(coinbase_coin.puzzle_hash)) + byte_array_tx.append(bytearray(fees_coin.puzzle_hash)) + + bip158: PyBIP158 = PyBIP158(byte_array_tx) + encoded = bytes(bip158.GetEncoded()) + + removal_merkle_set = MerkleSet() + addition_merkle_set = MerkleSet() + + tx_additions.append(coinbase_coin) + tx_additions.append(fees_coin) + + # Create removal Merkle set + for coin_name in tx_removals: + removal_merkle_set.add_already_hashed(coin_name) + + # Create addition Merkle set + puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {} + for coin in tx_additions: + if coin.puzzle_hash in puzzlehash_coin_map: + puzzlehash_coin_map[coin.puzzle_hash].append(coin) + else: + puzzlehash_coin_map[coin.puzzle_hash] = [coin] + + # Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash + for puzzle, coins in puzzlehash_coin_map.items(): + addition_merkle_set.add_already_hashed(puzzle) + addition_merkle_set.add_already_hashed(hash_coin_list(coins)) + + additions_root = addition_merkle_set.get_root() + removal_root = removal_merkle_set.get_root() + header_data: HeaderData = HeaderData( height, prev_header_hash, timestamp, - bytes([0] * 32), + encoded, proof_of_space.get_hash(), body.get_hash(), uint64(prev_weight + difficulty), uint64(prev_iters + number_iters), - bytes([0] * 32), - bytes([0] * 32), + additions_root, + removal_root, ) header_hash_sig: PrependSignature = plot_sk.sign_prepend(header_data.get_hash()) diff --git a/tests/test_blockchain.py b/tests/full_node/test_blockchain.py similarity index 94% rename from tests/test_blockchain.py rename to tests/full_node/test_blockchain.py index 03de38d6..fc7279d9 100644 --- a/tests/test_blockchain.py +++ b/tests/full_node/test_blockchain.py @@ -6,15 +6,14 @@ from pathlib import Path import pytest from blspy import PrivateKey -from src.blockchain import Blockchain, ReceiveBlockResult -from src.consensus.constants import constants -from src.store import FullNodeStore +from src.full_node.blockchain import Blockchain, ReceiveBlockResult +from src.full_node.store import FullNodeStore from src.types.body import Body from src.types.full_block import FullBlock -from src.types.hashable.Coin import Coin +from src.types.hashable.coin import Coin from src.types.header import Header, HeaderData from src.types.proof_of_space import ProofOfSpace -from src.coin_store import CoinStore +from src.full_node.coin_store import CoinStore from src.util.ints import uint8, uint64 from tests.block_tools import BlockTools @@ -29,6 +28,7 @@ test_constants: Dict[str, Any] = { "DIFFICULTY_EPOCH": 12, # The number of blocks per epoch "DIFFICULTY_WARP_FACTOR": 4, # DELAY divides EPOCH in order to warp efficiently. "DIFFICULTY_DELAY": 3, # EPOCH / WARP_FACTOR + "VDF_IPS_STARTING": 50, } test_constants["GENESIS_BLOCK"] = bytes( bt.create_genesis_block(test_constants, bytes([0] * 32), b"0") @@ -47,7 +47,7 @@ class TestGenesisBlock: unspent_store = await CoinStore.create(Path("blockchain_test.db")) store = await FullNodeStore.create(Path("blockchain_test.db")) await store._clear_database() - bc1 = await Blockchain.create(unspent_store, store) + bc1 = await Blockchain.create(unspent_store, store, test_constants) assert len(bc1.get_current_tips()) == 1 genesis_block = bc1.get_current_tips()[0] assert genesis_block.height == 0 @@ -90,7 +90,7 @@ class TestBlockValidation: blocks[9].header.data.height, bytes([1] * 32), blocks[9].header.data.timestamp, - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.body_hash, blocks[9].header.data.weight, @@ -117,7 +117,7 @@ class TestBlockValidation: blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, blocks[9].header.data.timestamp - 1000, - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.body_hash, blocks[9].header.data.weight, @@ -141,7 +141,7 @@ class TestBlockValidation: blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, uint64(int(time.time() + 3600 * 3)), - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.body_hash, blocks[9].header.data.weight, @@ -167,7 +167,7 @@ class TestBlockValidation: blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, blocks[9].header.data.timestamp, - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, bytes([1] * 32), blocks[9].header.data.weight, @@ -268,7 +268,7 @@ class TestBlockValidation: assert diff_27 > diff_26 assert (diff_27 / diff_26) <= test_constants["DIFFICULTY_FACTOR"] - assert (b.get_next_ips(blocks[1])) == constants["VDF_IPS_STARTING"] + assert (b.get_next_ips(blocks[1])) == test_constants["VDF_IPS_STARTING"] assert (b.get_next_ips(blocks[24])) == (b.get_next_ips(blocks[23])) assert (b.get_next_ips(blocks[25])) == (b.get_next_ips(blocks[24])) assert (b.get_next_ips(blocks[26])) > (b.get_next_ips(blocks[25])) @@ -292,7 +292,7 @@ class TestReorgs: assert b.get_current_tips()[0].height == 100 blocks_reorg_chain = bt.get_consecutive_blocks( - test_constants, 30, blocks[:90], 9, b"1" + test_constants, 30, blocks[:90], 9, b"2" ) for i in range(1, len(blocks_reorg_chain)): reorg_block = blocks_reorg_chain[i] @@ -321,9 +321,10 @@ class TestReorgs: # Reorg from genesis blocks_reorg_chain = bt.get_consecutive_blocks( - test_constants, 21, [blocks[0]], 9, b"1" + test_constants, 21, [blocks[0]], 9, b"3" ) for i in range(1, len(blocks_reorg_chain)): + print("I", i) reorg_block = blocks_reorg_chain[i] result, removed = await b.receive_block(reorg_block) if reorg_block.height == 0: @@ -336,7 +337,7 @@ class TestReorgs: # Reorg back to original branch blocks_reorg_chain_2 = bt.get_consecutive_blocks( - test_constants, 3, blocks[:-1], 9, b"3" + test_constants, 3, blocks[:-1], 9, b"4" ) result, _ = await b.receive_block(blocks_reorg_chain_2[20]) assert result == ReceiveBlockResult.ADDED_AS_ORPHAN @@ -389,8 +390,6 @@ class TestReorgs: await b.receive_block(blocks[i]) header_hashes = b.get_header_hashes(blocks[-1].header_hash) assert len(header_hashes) == 6 - print(header_hashes) - print([block.header_hash for block in blocks]) assert header_hashes == [block.header_hash for block in blocks] await unspent_store.close() diff --git a/tests/test_blockchain_transactions.py b/tests/full_node/test_blockchain_transactions.py similarity index 99% rename from tests/test_blockchain_transactions.py rename to tests/full_node/test_blockchain_transactions.py index 41f2fe04..b147c583 100644 --- a/tests/test_blockchain_transactions.py +++ b/tests/full_node/test_blockchain_transactions.py @@ -4,13 +4,13 @@ from typing import Optional import pytest from clvm.casts import int_to_bytes -from src.types.ConditionVarPair import ConditionVarPair +from src.types.condition_var_pair import ConditionVarPair from src.types.condition_opcodes import ConditionOpcode from src.util.bundle_tools import best_solution_program from src.server.outbound_message import OutboundMessage from src.protocols import full_node_protocol from src.types.full_block import FullBlock -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.spend_bundle import SpendBundle from src.util.ConsensusError import Err from src.util.ints import uint64 from tests.setup_nodes import setup_two_nodes, test_constants, bt diff --git a/tests/test_full_node.py b/tests/full_node/test_full_node.py similarity index 99% rename from tests/test_full_node.py rename to tests/full_node/test_full_node.py index 10dc938d..bbf85c42 100644 --- a/tests/test_full_node.py +++ b/tests/full_node/test_full_node.py @@ -11,10 +11,10 @@ from src.protocols import timelord_protocol from src.types.peer_info import PeerInfo from src.types.full_block import FullBlock from src.types.proof_of_space import ProofOfSpace -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.spend_bundle import SpendBundle from src.util.bundle_tools import best_solution_program from src.util.ints import uint16, uint32, uint64, uint8 -from src.types.ConditionVarPair import ConditionVarPair +from src.types.condition_var_pair import ConditionVarPair from src.types.condition_opcodes import ConditionOpcode from tests.setup_nodes import setup_two_nodes, test_constants, bt from tests.wallet_tools import WalletTool diff --git a/tests/test_full_sync.py b/tests/full_node/test_full_sync.py similarity index 100% rename from tests/test_full_sync.py rename to tests/full_node/test_full_sync.py diff --git a/tests/test_mempool.py b/tests/full_node/test_mempool.py similarity index 99% rename from tests/test_mempool.py rename to tests/full_node/test_mempool.py index ddf7d3a0..cc66330b 100644 --- a/tests/test_mempool.py +++ b/tests/full_node/test_mempool.py @@ -5,7 +5,7 @@ import pytest from src.server.outbound_message import OutboundMessage from src.protocols import full_node_protocol -from src.types.ConditionVarPair import ConditionVarPair +from src.types.condition_var_pair import ConditionVarPair from src.types.condition_opcodes import ConditionOpcode from src.util.ints import uint64 from tests.setup_nodes import setup_two_nodes, test_constants, bt @@ -43,7 +43,6 @@ class TestMempool: full_node_1, full_node_2, server_1, server_2 = two_nodes block = blocks[1] - print(f"block coinbase: {block.body.coinbase.name()}") async for _ in full_node_1.respond_block( full_node_protocol.RespondBlock(block) ): @@ -113,7 +112,6 @@ class TestMempool: outbound_2: OutboundMessage = _ # Maybe transaction means that it's accepted in mempool assert outbound_2.message.function == "new_transaction" - print(blocks[1].body.coinbase.name()) sb = full_node_1.mempool_manager.get_spendbundle(spend_bundle.name()) assert sb is spend_bundle diff --git a/tests/test_node_load.py b/tests/full_node/test_node_load.py similarity index 100% rename from tests/test_node_load.py rename to tests/full_node/test_node_load.py diff --git a/tests/test_store.py b/tests/full_node/test_store.py similarity index 97% rename from tests/test_store.py rename to tests/full_node/test_store.py index 165ea166..fc513c51 100644 --- a/tests/test_store.py +++ b/tests/full_node/test_store.py @@ -6,8 +6,7 @@ import sqlite3 import random import pytest -from src.consensus.constants import constants -from src.store import FullNodeStore +from src.full_node.store import FullNodeStore from src.types.full_block import FullBlock from src.types.sized_bytes import bytes32 from src.util.ints import uint32, uint64 @@ -58,7 +57,7 @@ class TestStore: try: await db._clear_database() - genesis = FullBlock.from_bytes(constants["GENESIS_BLOCK"]) + genesis = FullBlock.from_bytes(test_constants["GENESIS_BLOCK"]) # Save/get block for block in blocks: diff --git a/tests/full_node/test_transactions.py b/tests/full_node/test_transactions.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_unspent.py b/tests/full_node/test_unspent.py similarity index 98% rename from tests/test_unspent.py rename to tests/full_node/test_unspent.py index 1dd3b43d..f43c69cf 100644 --- a/tests/test_unspent.py +++ b/tests/full_node/test_unspent.py @@ -4,9 +4,9 @@ from pathlib import Path import pytest -from src.blockchain import Blockchain, ReceiveBlockResult -from src.store import FullNodeStore -from src.coin_store import CoinStore +from src.full_node.blockchain import Blockchain, ReceiveBlockResult +from src.full_node.store import FullNodeStore +from src.full_node.coin_store import CoinStore from tests.block_tools import BlockTools bt = BlockTools() diff --git a/tests/keys.py b/tests/keys.py index 13031798..8965731b 100644 --- a/tests/keys.py +++ b/tests/keys.py @@ -1,7 +1,7 @@ import blspy -from src.types.hashable.CoinSolution import CoinSolution -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.coin_solution import CoinSolution +from src.types.hashable.spend_bundle import SpendBundle from src.wallet.BLSPrivateKey import BLSPrivateKey from src.wallet.keychain import Keychain diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 238000a0..3655b9af 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1,37 +1,11 @@ import asyncio -from typing import Any, Dict -from pathlib import Path import pytest -from src.blockchain import Blockchain, ReceiveBlockResult -from src.mempool_manager import MempoolManager -from src.store import FullNodeStore -from src.full_node import FullNode -from src.server.connection import NodeType -from src.server.server import ChiaServer -from src.coin_store import CoinStore -from tests.block_tools import BlockTools from src.rpc.rpc_server import start_rpc_server +from src.protocols import full_node_protocol from src.rpc.rpc_client import RpcClient -from src.util.config import load_config - - -bt = BlockTools() - -test_constants: Dict[str, Any] = { - "DIFFICULTY_STARTING": 5, - "DISCRIMINANT_SIZE_BITS": 32, - "BLOCK_TIME_TARGET": 10, - "MIN_BLOCK_TIME": 2, - "DIFFICULTY_FACTOR": 3, - "DIFFICULTY_EPOCH": 12, # The number of blocks per epoch - "DIFFICULTY_WARP_FACTOR": 4, # DELAY divides EPOCH in order to warp efficiently. - "DIFFICULTY_DELAY": 3, # EPOCH / WARP_FACTOR -} -test_constants["GENESIS_BLOCK"] = bytes( - bt.create_genesis_block(test_constants, bytes([0] * 32), b"0") -) +from tests.setup_nodes import setup_two_nodes, test_constants, bt @pytest.fixture(scope="module") @@ -41,47 +15,28 @@ def event_loop(): class TestRpc: + @pytest.fixture(scope="function") + async def two_nodes(self): + async for _ in setup_two_nodes(): + yield _ + @pytest.mark.asyncio - async def test1(self): - test_node_1_port = 21234 - test_node_2_port = 21235 - test_rpc_port = 21236 - db_filename = Path("blockchain_test.db") + async def test1(self, two_nodes): + num_blocks = 10 + test_rpc_port = 21522 + full_node_1, full_node_2, server_1, server_2 = two_nodes + blocks = bt.get_consecutive_blocks(test_constants, num_blocks, [], 10) - if db_filename.exists(): - db_filename.unlink() - store = await FullNodeStore.create(db_filename) - await store._clear_database() - blocks = bt.get_consecutive_blocks(test_constants, 10, [], 10) - unspent_store = await CoinStore.create("blockchain_test.db") - mempool_manager = MempoolManager(unspent_store) - - b: Blockchain = await Blockchain.create(unspent_store, store, test_constants) - await store.add_block(blocks[0]) - for i in range(1, len(blocks)): - assert (await b.receive_block(blocks[i]))[ - 0 - ] == ReceiveBlockResult.ADDED_TO_HEAD - await store.add_block(blocks[i]) - - config = load_config("config.yaml", "full_node") - full_node_1 = FullNode(store, b, config, mempool_manager, unspent_store) - server_1 = ChiaServer(test_node_1_port, full_node_1, NodeType.FULL_NODE) - _ = await server_1.start_server("127.0.0.1", None) - full_node_1._set_server(server_1) + for i in range(1, num_blocks): + async for _ in full_node_1.respond_block( + full_node_protocol.RespondBlock(blocks[i]) + ): + pass def stop_node_cb(): full_node_1._shutdown() server_1.close_all() - unspent_store2 = await CoinStore.create("blockchain_test_2.db") - mempool_manager2 = MempoolManager(unspent_store2) - full_node_2 = FullNode(store, b, config, mempool_manager2, unspent_store2) - server_2 = ChiaServer(test_node_2_port, full_node_2, NodeType.FULL_NODE) - full_node_2._set_server(server_2) - - _ = await server_2.start_server("127.0.0.1", None) - rpc_cleanup = await start_rpc_server(full_node_1, stop_node_cb, test_rpc_port) try: @@ -94,22 +49,20 @@ class TestRpc: assert state["ips"] > 0 block = await client.get_block(state["lca"].header_hash) - print([b.header_hash for b in blocks]) - print(block.header_hash) - assert block == blocks[8] + assert block == blocks[7] assert (await client.get_block(bytes([1] * 32))) is None header = await client.get_header(state["lca"].header_hash) - assert header == blocks[8].header + assert header == blocks[7].header coins = await client.get_unspent_coins( blocks[-1].body.coinbase.puzzle_hash, blocks[-1].header_hash ) - assert len(coins) == 22 + assert len(coins) == 16 coins_lca = await client.get_unspent_coins( blocks[-1].body.coinbase.puzzle_hash ) - assert len(coins_lca) == 18 + assert len(coins_lca) == 16 assert len(await client.get_connections()) == 0 @@ -123,27 +76,11 @@ class TestRpc: await asyncio.sleep(2) # Allow server to start except AssertionError: # Checks that the RPC manages to stop the node - await client.stop_node() client.close() await client.await_closed() - server_2.close_all() - await server_1.await_closed() - await server_2.await_closed() await rpc_cleanup() - await store.close() - Path("blockchain_test.db").unlink() - Path("blockchain_test_2.db").unlink() raise - await client.stop_node() client.close() await client.await_closed() - server_2.close_all() - await server_1.await_closed() - await server_2.await_closed() await rpc_cleanup() - await store.close() - await unspent_store.close() - await unspent_store2.close() - Path("blockchain_test.db").unlink() - Path("blockchain_test_2.db").unlink() diff --git a/tests/setup_nodes.py b/tests/setup_nodes.py index d47830d2..38c39dfa 100644 --- a/tests/setup_nodes.py +++ b/tests/setup_nodes.py @@ -2,14 +2,14 @@ from typing import Any, Dict from pathlib import Path import asyncio -from src.blockchain import Blockchain -from src.mempool_manager import MempoolManager -from src.store import FullNodeStore -from src.full_node import FullNode +from src.full_node.blockchain import Blockchain +from src.full_node.mempool_manager import MempoolManager +from src.full_node.store import FullNodeStore +from src.full_node.full_node import FullNode from src.server.connection import NodeType from src.server.server import ChiaServer from src.types.full_block import FullBlock -from src.coin_store import CoinStore +from src.full_node.coin_store import CoinStore from tests.block_tools import BlockTools from src.types.hashable.BLSSignature import BLSPublicKey from src.util.config import load_config @@ -68,7 +68,13 @@ async def setup_full_node(db_name, port, introducer_port=None, dic={}): config["introducer_peer"]["host"] = "127.0.0.1" config["introducer_peer"]["port"] = introducer_port full_node_1 = FullNode( - store_1, b_1, config, mempool_1, unspent_store_1, f"full_node_{port}" + store_1, + b_1, + config, + mempool_1, + unspent_store_1, + f"full_node_{port}", + test_constants_copy, ) server_1 = ChiaServer(port, full_node_1, NodeType.FULL_NODE) _ = await server_1.start_server(config["host"], full_node_1._on_connect) @@ -117,8 +123,11 @@ async def setup_farmer(port, dic={}): "pool_sks": [bytes(pool_sk).hex()], "pool_target": pool_target.hex(), } + test_constants_copy = test_constants.copy() + for k in dic.keys(): + test_constants_copy[k] = dic[k] - farmer = Farmer(config, key_config) + farmer = Farmer(config, key_config, test_constants_copy) server = ChiaServer(port, farmer, NodeType.FARMER) _ = await server.start_server(config["host"], farmer._on_connect) @@ -143,8 +152,11 @@ async def setup_introducer(port, dic={}): async def setup_timelord(port, dic={}): config = load_config("config.yaml", "timelord") + test_constants_copy = test_constants.copy() + for k in dic.keys(): + test_constants_copy[k] = dic[k] - timelord = Timelord(config) + timelord = Timelord(config, test_constants_copy["DISCRIMINANT_SIZE_BITS"]) server = ChiaServer(port, timelord, NodeType.TIMELORD) _ = await server.start_server(port, None) diff --git a/tests/test_filter.py b/tests/test_filter.py new file mode 100644 index 00000000..b152de3d --- /dev/null +++ b/tests/test_filter.py @@ -0,0 +1,57 @@ +import asyncio +from typing import List + +import pytest +from blspy import ExtendedPrivateKey +from chiabip158 import PyBIP158 + +from src.wallet.wallet_node import WalletNode +from tests.setup_nodes import setup_two_nodes, test_constants, bt + + +@pytest.fixture(scope="module") +def event_loop(): + loop = asyncio.get_event_loop() + yield loop + + +class TestFilter: + @pytest.fixture(scope="function") + async def two_nodes(self): + async for _ in setup_two_nodes({"COINBASE_FREEZE_PERIOD": 0}): + yield _ + + @pytest.mark.asyncio + async def test_basic_filter_test(self, two_nodes): + sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() + key_config = {"wallet_sk": sk} + wallet_node = await WalletNode.create({}, key_config) + wallet = wallet_node.wallet + await wallet_node.wallet_store._clear_database() + + num_blocks = 2 + blocks = bt.get_consecutive_blocks( + test_constants, + num_blocks, + [], + 10, + reward_puzzlehash=wallet.get_new_puzzlehash(), + ) + + for i in range(1, num_blocks): + byte_array_tx: List[bytes] = [] + block = blocks[i] + coinbase = bytearray(block.body.coinbase.puzzle_hash) + fee = bytearray(block.body.fees_coin.puzzle_hash) + byte_array_tx.append(coinbase) + byte_array_tx.append(fee) + + pl = PyBIP158(byte_array_tx) + present = pl.Match(coinbase) + fee_present = pl.Match(fee) + + assert present + assert fee_present + + await wallet_node.wallet_store.close() + await wallet_node.tx_store.close() diff --git a/tests/test_merkle_set.py b/tests/test_merkle_set.py new file mode 100644 index 00000000..270a89d7 --- /dev/null +++ b/tests/test_merkle_set.py @@ -0,0 +1,50 @@ +import asyncio + +import pytest + +from src.util.merkle_set import MerkleSet, confirm_included_already_hashed +from tests.setup_nodes import test_constants, bt +from tests.wallet_tools import WalletTool + + +@pytest.fixture(scope="module") +def event_loop(): + loop = asyncio.get_event_loop() + yield loop + + +class TestMerkleSet: + @pytest.mark.asyncio + async def test_basics(self): + wallet_tool = WalletTool() + + num_blocks = 10 + blocks = bt.get_consecutive_blocks( + test_constants, + num_blocks, + [], + 10, + reward_puzzlehash=wallet_tool.get_new_puzzlehash(), + ) + + merkle_set = MerkleSet() + for block in blocks: + merkle_set.add_already_hashed(block.body.coinbase.name()) + + for block in blocks: + result, proof = merkle_set.is_included_already_hashed( + block.body.coinbase.name() + ) + assert result is True + result_fee, proof_fee = merkle_set.is_included_already_hashed( + block.body.fees_coin.name() + ) + assert result_fee is False + validate_proof = confirm_included_already_hashed( + merkle_set.get_root(), block.body.coinbase.name(), proof + ) + validate_proof_fee = confirm_included_already_hashed( + merkle_set.get_root(), block.body.fees_coin.name(), proof_fee + ) + assert validate_proof is True + assert validate_proof_fee is False diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 78d14d20..7b264c2a 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -5,6 +5,7 @@ from blspy import ExtendedPrivateKey from src.protocols.wallet_protocol import RespondBody from src.wallet.wallet import Wallet +from src.wallet.wallet_node import WalletNode from tests.setup_nodes import setup_two_nodes, test_constants, bt @@ -25,8 +26,10 @@ class TestWallet: sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() key_config = {"wallet_sk": sk} - wallet = await Wallet.create({}, key_config) - await wallet.wallet_store._clear_database() + wallet_node = await WalletNode.create({}, key_config) + wallet = wallet_node.wallet + await wallet_node.wallet_store._clear_database() + await wallet_node.tx_store._clear_database() num_blocks = 10 blocks = bt.get_consecutive_blocks( @@ -39,11 +42,12 @@ class TestWallet: for i in range(1, num_blocks): a = RespondBody(blocks[i].body, blocks[i].height) - await wallet.received_body(a) + await wallet_node.received_body(a) assert await wallet.get_confirmed_balance() == 144000000000000 - await wallet.wallet_store.close() + await wallet_node.wallet_store.close() + await wallet_node.tx_store.close() @pytest.mark.asyncio async def test_wallet_make_transaction(self, two_nodes): @@ -52,10 +56,15 @@ class TestWallet: key_config = {"wallet_sk": sk} key_config_b = {"wallet_sk": sk_b} - wallet = await Wallet.create({}, key_config) - await wallet.wallet_store._clear_database() - wallet_b = await Wallet.create({}, key_config_b) - await wallet_b.wallet_store._clear_database() + wallet_node = await WalletNode.create({}, key_config) + wallet = wallet_node.wallet + await wallet_node.wallet_store._clear_database() + await wallet_node.tx_store._clear_database() + + wallet_node_b = await WalletNode.create({}, key_config_b) + wallet_b = wallet_node_b.wallet + await wallet_node_b.wallet_store._clear_database() + await wallet_node_b.tx_store._clear_database() num_blocks = 10 blocks = bt.get_consecutive_blocks( @@ -68,7 +77,7 @@ class TestWallet: for i in range(1, num_blocks): a = RespondBody(blocks[i].body, blocks[i].height) - await wallet.received_body(a) + await wallet_node.received_body(a) assert await wallet.get_confirmed_balance() == 144000000000000 @@ -83,5 +92,8 @@ class TestWallet: assert confirmed_balance == 144000000000000 assert unconfirmed_balance == confirmed_balance - 10 - await wallet.wallet_store.close() - await wallet_b.wallet_store.close() + await wallet_node.wallet_store.close() + await wallet_node.tx_store.close() + + await wallet_node_b.wallet_store.close() + await wallet_node_b.tx_store.close() diff --git a/tests/test_wallet_protocol.py b/tests/test_wallet_protocol.py deleted file mode 100644 index 5e88afe8..00000000 --- a/tests/test_wallet_protocol.py +++ /dev/null @@ -1,59 +0,0 @@ -import asyncio -import signal -import time - -import pytest -from blspy import ExtendedPrivateKey - -from src.protocols import full_node_protocol -from src.server.outbound_message import NodeType -from src.server.server import ChiaServer -from src.types.peer_info import PeerInfo -from src.wallet.wallet import Wallet -from tests.setup_nodes import setup_two_nodes, test_constants, bt - - -@pytest.fixture(scope="module") -def event_loop(): - loop = asyncio.get_event_loop() - yield loop - - -class TestWalletProtocol: - @pytest.fixture(scope="function") - async def two_nodes(self): - async for _ in setup_two_nodes({"COINBASE_FREEZE_PERIOD": 0}): - yield _ - - @pytest.mark.asyncio - async def test_wallet_connect(self, two_nodes): - num_blocks = 10 - blocks = bt.get_consecutive_blocks(test_constants, num_blocks, [], 10) - full_node_1, full_node_2, server_1, server_2 = two_nodes - - for i in range(1, num_blocks): - async for _ in full_node_1.respond_block( - full_node_protocol.RespondBlock(blocks[i]) - ): - pass - - sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() - key_config = {"wallet_sk": sk} - - wallet = await Wallet.create({}, key_config) - server = ChiaServer(8223, wallet, NodeType.WALLET) - - asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) - asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all) - - _ = await server.start_server("127.0.0.1", wallet._on_connect) - await asyncio.sleep(2) - full_node_peer = PeerInfo(server_1._host, server_1._port) - _ = await server.start_client(full_node_peer, None) - - start_unf = time.time() - while time.time() - start_unf < 3: - # TODO check if we've synced proof hashes and verified number of proofs - await asyncio.sleep(0.1) - - await wallet.wallet_store.close() diff --git a/tests/wallet_tools.py b/tests/wallet_tools.py index ec301071..2fa20e59 100644 --- a/tests/wallet_tools.py +++ b/tests/wallet_tools.py @@ -5,13 +5,13 @@ from clvm.casts import int_to_bytes, int_from_bytes from os import urandom from blspy import ExtendedPrivateKey -from src.types.ConditionVarPair import ConditionVarPair +from src.types.condition_var_pair import ConditionVarPair from src.types.condition_opcodes import ConditionOpcode -from src.types.hashable.Program import Program +from src.types.hashable.program import Program from src.types.hashable.BLSSignature import BLSSignature -from src.types.hashable.Coin import Coin -from src.types.hashable.CoinSolution import CoinSolution -from src.types.hashable.SpendBundle import SpendBundle +from src.types.hashable.coin import Coin +from src.types.hashable.coin_solution import CoinSolution +from src.types.hashable.spend_bundle import SpendBundle from src.util.condition_tools import ( conditions_by_opcode, hash_key_pairs_for_conditions_dict,