diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b137891..8b092c58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1 +1,121 @@ - +stages: + - test + - build +variables: + GIT_DEPTH: "3" + SIMPLECOV: "true" + RUST_BACKTRACE: "1" + RUSTFLAGS: "" + CARGOFLAGS: "" +cache: + key: "$CI_BUILD_STAGE/$CI_BUILD_REF_NAME" + untracked: true +linux-stable-test: + stage: test + image: parity/rust:gitlab-ci + only: + - beta + - tags + - stable + - triggers + - master + script: + - rustup default stable + - echo 'Running tests from /tools/test.sh' && echo -en 'gitlab_fold:start:tests' + - ./tools/test.sh + - echo -en 'gitlab_fold:end:tests' + tags: + - rust +linux-beta-test: + stage: test + image: parity/rust:gitlab-ci + only: + - beta + - tags + - stable + - triggers + - master + script: + - rustup default beta + - echo 'Running check' + - cargo check -j $(nproc) + - echo 'End check' + tags: + - rust + allow_failure: true +linux-nightly-test: + stage: test + image: parity/rust:gitlab-ci + only: + - beta + - tags + - stable + - triggers + - master + script: + - rustup default nightly + - echo 'Running check' + - cargo check -j $(nproc) + - echo 'End check' + tags: + - rust + allow_failure: true +linux-stable: + stage: build + image: parity/rust:gitlab-ci + only: + - beta + - tags + - stable + - triggers + - master + script: + - git submodule update --init + - echo 'Building in release mode' && echo -en 'gitlab_fold:start:build.release' + - rustup default stable + - cargo build -j $(nproc) -p pbtc --release + - strip target/release/pbtc + #- export SHA3=$(target/release/pbtc tools hash target/release/pbtc) + - md5sum target/release/pbtc > pbtc.md5 + - echo -en 'gitlab_fold:end:build.release' + - echo 'Running regtests from ./tools/compare-tool/pull-tests-be0eef7.jar' && echo -en 'gitlab_fold:start:regtests' + - ./tools/regtests.sh + - echo -en 'gitlab_fold:end:regtests' + - echo 'Running bench from /benhcer' && echo -en 'gitlab_fold:start:bench' + - ./tools/bench.sh + - echo -en 'gitlab_fold:end:bench' + - sh tools/deb_build.sh amd64 + - cp target/release/pbtc deb/usr/bin/pbtc + - export VER=$(grep -m 1 version Cargo.toml | awk '{print $3}' | tr -d '"' | tr -d "\n") + - dpkg-deb -b deb "pbtc_"$VER"_amd64.deb" + - md5sum "pbtc_"$VER"_amd64.deb" > "pbtc_"$VER"_amd64.deb.md5" + - aws configure set aws_access_key_id $s3_key + - aws configure set aws_secret_access_key $s3_secret + - if [[ $CI_BUILD_REF_NAME =~ ^(master|beta|stable|nightly)$ ]]; then export S3_BUCKET=parity-bitcoin-published; else export S3_BUCKET=parity-bitcoin-builds; fi + - aws s3 rm --recursive s3://$S3_BUCKET/$CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu + - aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu/pbtc --body target/release/pbtc + - aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu/pbtc.md5 --body pbtc.md5 + - aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu/"pbtc_"$VER"_amd64.deb" --body "pbtc_"$VER"_amd64.deb" + - aws s3api put-object --bucket $S3_BUCKET --key $CI_BUILD_REF_NAME/x86_64-unknown-linux-gnu/"pbtc_"$VER"_amd64.deb.md5" --body "pbtc_"$VER"_amd64.deb.md5" + tags: + - rust + artifacts: + paths: + - target/release/pbtc + - pbtc.md5 + name: "stable-x86_64-unknown-linux-gnu_parity_bitcoin" +docker-build: + stage: build + only: + - master + - tags + - triggers + before_script: + - docker info + script: + - if [ "$CI_BUILD_REF_NAME" == "beta-release" ]; then DOCKER_TAG="latest"; else DOCKER_TAG=$CI_BUILD_REF_NAME; fi + - docker login -u $Docker_Hub_User_Parity -p $Docker_Hub_Pass_Parity + - sh tools/docker_build.sh $DOCKER_TAG parity + - docker logout + tags: + - docker diff --git a/docker/hub/Dockerfile b/docker/hub/Dockerfile new file mode 100644 index 00000000..59348713 --- /dev/null +++ b/docker/hub/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu:14.04 +MAINTAINER Parity Technologies +WORKDIR /build +#ENV for build TAG +ARG BUILD_TAG +ENV BUILD_TAG ${BUILD_TAG:-master} +RUN echo $BUILD_TAG +# install tools and dependencies +RUN apt-get update && \ + apt-get install -y --force-yes --no-install-recommends \ + g++ \ + build-essential \ + curl \ + git \ + file \ + binutils \ + ca-certificates \ + libssl-dev \ + pkg-config \ + libudev-dev && \ +# install rustup + curl https://sh.rustup.rs -sSf | sh -s -- -y && \ +# rustup directory + PATH=/root/.cargo/bin:$PATH && \ +# show backtraces + RUST_BACKTRACE=1 && \ +# build pbtc-ubuntu + cd /build&&git clone https://github.com/paritytech/parity-bitcoin && \ + cd parity-bitcoin && \ + git pull&& \ + git checkout $BUILD_TAG && \ + cargo build --verbose --release && \ + strip /build/parity-bitcoin/target/release/pbtc && \ + file /build/parity-bitcoin/target/release/pbtc&&mkdir -p /pbtc-ubuntu&& \ + cp /build/parity-bitcoin/target/release/pbtc /pbtc-ubuntu/pbtc-ubuntu&& \ +#cleanup Docker image + rm -rf /root/.cargo&&rm -rf /root/.multirust&&rm -rf /root/.rustup&&rm -rf /build&&\ + apt-get purge -y \ + g++ \ + build-essential \ + curl \ + git \ + file \ + binutils \ + libssl-dev \ + pkg-config \ + libudev-dev && \ + rm -rf /var/lib/apt/lists/* +# setup ENTRYPOINT +EXPOSE 8333 18333 8332 18332 +ENTRYPOINT ["/pbtc-ubuntu/pbtc-ubuntu"] diff --git a/sync/src/utils/bloom_filter.rs b/sync/src/utils/bloom_filter.rs index 973708c9..5344fd47 100644 --- a/sync/src/utils/bloom_filter.rs +++ b/sync/src/utils/bloom_filter.rs @@ -159,7 +159,8 @@ impl BloomFilterData { for hash_function_idx in 0..self.hash_functions_num { let murmur_seed = hash_function_idx.overflowing_mul(SEED_OFFSET).0.overflowing_add(self.tweak).0; let murmur_hash = murmur3_32(&mut data.as_ref(), murmur_seed) as usize % self.filter.len(); - if !self.filter.get(murmur_hash).expect("murmur_hash is result of mod operation by filter len; qed") { + let index = (murmur_hash & !7usize) | ((murmur_hash & 7) ^ 7); + if !self.filter.get(index).expect("murmur_hash is result of mod operation by filter len; qed") { return false; } } @@ -171,7 +172,8 @@ impl BloomFilterData { for hash_function_idx in 0..self.hash_functions_num { let murmur_seed = hash_function_idx.overflowing_mul(SEED_OFFSET).0.overflowing_add(self.tweak).0; let murmur_hash = murmur3_32(&mut data.as_ref(), murmur_seed) as usize % self.filter.len(); - self.filter.set(murmur_hash, true); + let index = (murmur_hash & !7usize) | ((murmur_hash & 7) ^ 7); + self.filter.set(index, true); } } } @@ -284,4 +286,18 @@ mod tests { assert!(filter.filter_transaction(&tx1)); assert!(!filter.filter_transaction(&tx2)); } + + #[test] + fn bloom_filter_data_works_on_address() { + use message::{types, deserialize_payload}; + + // real world message + let payload: Bytes = "fd77078732c06c257f2818a0d14738804d6a18a0f0a9800e039be473286d591f48040c99e96d2e0cd8480b03c0823fdd79310e0e04c86314ae24cabaa931f39852c813172e524c4b231020557dda2023130b0f1f309de822014bf6d4305d238ea013b8539e9c138053076a22020335bc1042e543f260787993a07a155a7a560c1475f75ec109b73822491c5880924303a5545ffc465132960fe40a8822010f919c38910f31c546be6940090f0476a02570de8d28258ee8081dea073829386a9de01b0070348f0f2fb48c6a1b4e92292de7b60dbafb8df3a6cab6a824e50005744018b384f080c8265152e406b1c85906d5325d1c83ac880d02214ad401ddc07657002c47708e338020bd14fcc147bfe49335647062074a4d3276e0d3a5110826a05d4fb025129f48c26e5819dd878d851b84802b7a211097813ef952410390c95bb294a7a8faca3667f0068489a69d9c9e8405c035442874a6a8c448c60600834a22041ce36a8065b086469bbb1b46d326b7a9923054ad4e32e4b7231aa203c5acab1f5821b92d2f00728819e587e1e6ff9fa6e66ff52fb54bce3648a7b043cbd19469aa5af0891eb4979def822f06522f080830b411545b4240e01195b0f0962e628050f0f8290269c45c20aa16559d17ceca68b81a21370909a1086614531577600ad8480ae1023c9173260bc6385c0e2d46806c05401a17ac969edfe65fd4000007e1c8a13ac651922edfa5a3235a0bcc10cc0067b4404d64260aea022391b007d3010c8380fda9c821d37258e47a0ab6485baff88876ba55cd190edf730970875f75522010e40f8c28e4872a2065c96390041bef9273a6a1c05d2bd28193b205b2cf25e44c5290926a4d392450b3135a0db7cc2c0696384eee68a65a4280ac3c44207264243fd18795c69588f4e418a26d4e21d1cee894325a2fc869b03fec68fd3e86b31998f9f9a6402dd672227451b1b5419a89c90ae96b9292a1ca83848bc026dcd00ec740342496730a7ab8e48200c7ea240d0e34a07890a9203938ee188475b3d6dd2f50399019c249536955894917fa3acc400ce04ec42d5e89e2d48aa085ae129226d0ea2a4d0038db88fd5ec0688768cea449171280438e8f5164d8682c40b91a2dab042800b1f312b22460e4905dccee59842a2bc9f807d542e08388fec013696d281c0356880040b0610ac4cb148a95a5924875891b2217040bbab7bba21f14898203ae87153206601c20c484f072216c1714ac5ded41a6d213fc09962f195129ac09d85dc05501a773163646021100481cb385a0fdb1609e8d1bc042942f169884e60482ae2004924fc2eb0b7061855858b5e54a4517352084778f38c09f1f004431da4531059c1296436d4c89e8839d34506a27b07c94fa80d2a0a9b73208a79982f0a5d16e0b723d38816a6a666c09bceae5a46d8a032248302224ae43c7e4801004ea6671c6d08142b67e27269a4ac6418748d74f481fc2dbbac5852afc645026cb462f790347d32a26e0b88209f2168110eb4e88394dd0f3bc27958994803238059d581d8cda73493a994c433fad902e20093ad40b1570543af928bc4830c3976d2802635400c0ac5a25833008f00b8bb691c1d9ce026373d05ac03fa6851402e05c6f109fa1754da54a20aa9f4f4782e68966a5113ae141d495d2a53d64bb6a023616b243e0333e2e0c65f2078559a98f48e8637f7b2ff326572a32532e2ec9e9651c092c52a522a5120249a159fa49d56d304806b502c425e3981275409f38b20418bc8206d21e884401c05c3c7d0b7404cf2a97706092b5a818638122ce750ce8780812900f501f4f02cb90fd5ea615e611006e010920072d08ca01b535741460c9aa1ab567ac2f79a004400e523fdcf95320bc08a37f54aaa01b2a2d0aa343ec13205131124b445a2c1b9a7542c63c6ced549447462099c12a9c613838995d718849650300056845311b2c93a1d35a66622466e0a3cef68594faa11021751c0e5358027721d2a2362fc9353655680c80fa5cad35685a0454932251aa121cb50583ae987c2a6e8009f3342048019509760b7e233de62cdcb636344d8a012af9f61a539ec66801c46a56b3d8ea6825e95430d1cc71fcc0bc977e4ea27f83e284cab0aea0a5085e67039901252a722054e33168a89d160a5908a325b63654016da1e94450548bac84a1b22fcc92ce7e2018967755711480e9048f5ded20b5d3960f21b559f1a0be84a53721f7b0f283d7d1bc2bd7f5d7550d3213814eb56e13f0b106acc07b05ece2c518117e934409843f1f889c2d84845c540514badb4ca00864e8bde78e50b0837478104c018cd5996977196e4f064002480a2761489000984d44d0077df65696f306930c893c50b08516e9ea82e02eb0400bc3d1adade23161b45e5210e08e981568f8af232bdf0f3c460349a8800c9f2c510eecc8ccee39e8b0898d329560aaf4d9594a551186423f79aa6806ce8c541506283a54e8859a840814012cad0289627f8659658218f6e58926af0849b4b23b40ac76280061b90c940f71617e0397ea145968250b1060608e4002432021195635dc52e0495c69fa67768a4a89ec32206fa30f62a85503de8c79df940f808c0de2f1723c0c84d89f317c4c1287a40759946d9cc8c43044a817dd6bb8ed326e4ab800fd8815482910de3cc360d40080a8d956e049ec6d1000000000943a3102".parse().unwrap(); + let message: types::FilterLoad = deserialize_payload(&payload, 70001).unwrap(); + // containing this address + let address: Bytes = "BA99435B59DEBDBB5E207C0B33FF55752DBC5EFF".parse().unwrap(); + + let bloom = BloomFilterData::with_filter_load(message); + assert!(bloom.contains(address.as_slice())); + } } diff --git a/tools/deb-build.sh b/tools/deb-build.sh new file mode 100644 index 00000000..7a2b4f41 --- /dev/null +++ b/tools/deb-build.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e # fail on any error +set -u # treat unset variables as error +rm -rf deb +#create DEBIAN files +mkdir -p deb/usr/bin/ +mkdir -p deb/DEBIAN +#create copyright, docs, compat +cp LICENSE deb/DEBIAN/copyright +echo "https://github.com/paritytech/parity-bitcoin/" >> deb/DEBIAN/docs +echo "8" >> deb/DEBIAN/compat +#create control file +control=deb/DEBIAN/control +echo "Package: pbtc" >> $control +version=`grep -m 1 version Cargo.toml | awk '{print $3}' | tr -d '"' | tr -d "\n"` +echo "Version: $version" >> $control +echo "Source: pbtc" >> $control +echo "Section: science" >> $control +echo "Priority: extra" >> $control +echo "Maintainer: Parity Technologies " >> $control +echo "Build-Depends: debhelper (>=9)" >> $control +echo "Standards-Version: 3.9.5" >> $control +echo "Homepage: https://parity.io" >> $control +echo "Vcs-Git: git://github.com/paritytech/parity-bitcoin.git" >> $control +echo "Vcs-Browser: https://github.com/paritytech/parity-bitcoin" >> $control +echo "Architecture: $1" >> $control +echo "Depends: libssl1.0.0 (>=1.0.0)" >> $control +echo "Description: Bitcoin network client by Parity Technologies" >> $control +#build .deb package + +exit diff --git a/tools/deb_build.sh b/tools/deb_build.sh new file mode 100644 index 00000000..7a2b4f41 --- /dev/null +++ b/tools/deb_build.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e # fail on any error +set -u # treat unset variables as error +rm -rf deb +#create DEBIAN files +mkdir -p deb/usr/bin/ +mkdir -p deb/DEBIAN +#create copyright, docs, compat +cp LICENSE deb/DEBIAN/copyright +echo "https://github.com/paritytech/parity-bitcoin/" >> deb/DEBIAN/docs +echo "8" >> deb/DEBIAN/compat +#create control file +control=deb/DEBIAN/control +echo "Package: pbtc" >> $control +version=`grep -m 1 version Cargo.toml | awk '{print $3}' | tr -d '"' | tr -d "\n"` +echo "Version: $version" >> $control +echo "Source: pbtc" >> $control +echo "Section: science" >> $control +echo "Priority: extra" >> $control +echo "Maintainer: Parity Technologies " >> $control +echo "Build-Depends: debhelper (>=9)" >> $control +echo "Standards-Version: 3.9.5" >> $control +echo "Homepage: https://parity.io" >> $control +echo "Vcs-Git: git://github.com/paritytech/parity-bitcoin.git" >> $control +echo "Vcs-Browser: https://github.com/paritytech/parity-bitcoin" >> $control +echo "Architecture: $1" >> $control +echo "Depends: libssl1.0.0 (>=1.0.0)" >> $control +echo "Description: Bitcoin network client by Parity Technologies" >> $control +#build .deb package + +exit diff --git a/tools/docker_build.sh b/tools/docker_build.sh index c98f3013..88fdcca2 100755 --- a/tools/docker_build.sh +++ b/tools/docker_build.sh @@ -1,3 +1,6 @@ #!/bin/bash -docker build --force-rm -t pbtc-ubuntu -f docker/ubuntu/Dockerfile . +cd docker/hub +if [ "$1" == "latest" ]; then DOCKER_BUILD_TAG="beta-release"; fi +docker build --build-arg BUILD_TAG=$DOCKER_BUILD_TAG --no-cache=true --tag $2/pbtc-ubuntu:$1 . +docker push $2/pbtc-ubuntu:$1