zebrad: use hedged requests in sync
The hedge middleware implements hedged requests, as described in _The Tail At Scale_. The idea is that we auto-tune our retry logic according to the actual network conditions, pre-emptively retrying requests that exceed some latency percentile. This would hopefully solve the problem where our timeouts are too long on mainnet and too slow on testnet.
This commit is contained in:
Henry de Valence
parent
5f229d1475
commit
12d25159c6
|
|
@ -38,7 +38,7 @@ dependencies = [
|
|||
"ident_case",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
|
|
@ -160,6 +160,15 @@ dependencies = [
|
|||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.12.3"
|
||||
|
|
@ -390,7 +399,7 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
|
||||
dependencies = [
|
||||
"nom",
|
||||
"nom 5.1.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -521,6 +530,12 @@ dependencies = [
|
|||
"proc-macro-hack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_fn"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.1.5"
|
||||
|
|
@ -549,11 +564,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-channel 0.4.4",
|
||||
"crossbeam-deque 0.7.3",
|
||||
"crossbeam-epoch 0.8.2",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-utils 0.7.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.6.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -562,21 +586,42 @@ version = "0.4.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"maybe-uninit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-utils 0.8.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-epoch 0.8.2",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"maybe-uninit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-epoch 0.9.0",
|
||||
"crossbeam-utils 0.8.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.8.2"
|
||||
|
|
@ -585,13 +630,27 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
|
|||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 0.1.10",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"lazy_static",
|
||||
"maybe-uninit",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0f606a85340376eef0d6d8fec399e6d4a544d648386c6645eb6d0653b27d9f"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"const_fn",
|
||||
"crossbeam-utils 0.8.0",
|
||||
"lazy_static",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.2.3"
|
||||
|
|
@ -599,10 +658,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"maybe-uninit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.7.2"
|
||||
|
|
@ -614,6 +683,18 @@ dependencies = [
|
|||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 1.0.0",
|
||||
"const_fn",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.2"
|
||||
|
|
@ -627,7 +708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "7fbaabec2c953050352311293be5c6aba8e141ba19d6811862b232d6fd020484"
|
||||
dependencies = [
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -665,7 +746,7 @@ dependencies = [
|
|||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"strsim 0.9.3",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -676,7 +757,7 @@ checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
|
|||
dependencies = [
|
||||
"darling_core",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -731,7 +812,7 @@ checksum = "adc2ab4d5a16117f9029e9a6b5e4e79f4c67f6519bc134210d4d4a04ba31f41b"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -836,6 +917,18 @@ dependencies = [
|
|||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"crc32fast",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
|
|
@ -937,7 +1030,7 @@ dependencies = [
|
|||
"proc-macro-hack",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1022,7 +1115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
"version_check 0.9.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1088,7 +1181,7 @@ checksum = "90454ce4de40b7ca6a8968b5ef367bdab48413962588d0d2b1638d60090c35d7"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1122,7 +1215,11 @@ version = "6.3.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d331ebcdbca4acbefe5da8c3299b2e246f198a8294cc5163354e743398b89d"
|
||||
dependencies = [
|
||||
"base64 0.10.1",
|
||||
"byteorder",
|
||||
"crossbeam-channel 0.3.9",
|
||||
"flate2",
|
||||
"nom 4.2.3",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
|
|
@ -1244,7 +1341,7 @@ dependencies = [
|
|||
"rand_xoshiro",
|
||||
"sized-chunks",
|
||||
"typenum",
|
||||
"version_check",
|
||||
"version_check 0.9.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1536,7 +1633,7 @@ checksum = "ce0e4f69639ccc0c6b2f0612164f9817349eb25545ed1ffb5ef3e1e1c1d220b4"
|
|||
dependencies = [
|
||||
"arc-swap",
|
||||
"atomic-shim",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"im",
|
||||
"metrics",
|
||||
"metrics-core",
|
||||
|
|
@ -1556,7 +1653,7 @@ version = "0.3.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d11f8090a8886339f9468a04eeea0711e4cf27538b134014664308041307a1c5"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-epoch 0.8.2",
|
||||
"serde",
|
||||
]
|
||||
|
||||
|
|
@ -1651,6 +1748,16 @@ version = "0.1.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "4.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"version_check 0.1.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "5.1.2"
|
||||
|
|
@ -1658,7 +1765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"version_check",
|
||||
"version_check 0.9.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1839,7 +1946,7 @@ checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1892,8 +1999,8 @@ dependencies = [
|
|||
"proc-macro-error-attr",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"version_check",
|
||||
"syn 1.0.46",
|
||||
"version_check 0.9.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1904,7 +2011,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"version_check",
|
||||
"version_check 0.9.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2114,25 +2221,25 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.4.1"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcf6960dc9a5b4ee8d3e4c5787b4a112a8818e0290a42ff664ad60692fdf2032"
|
||||
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-deque 0.8.0",
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8c4fec834fb6e6d2dd5eece3c7b432a52f0ba887cf40e595190c4107edc08bf"
|
||||
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-channel 0.5.0",
|
||||
"crossbeam-deque 0.8.0",
|
||||
"crossbeam-utils 0.8.0",
|
||||
"lazy_static",
|
||||
"num_cpus",
|
||||
]
|
||||
|
|
@ -2241,10 +2348,10 @@ version = "0.8.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.12.3",
|
||||
"blake2b_simd",
|
||||
"constant_time_eq",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-utils 0.7.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2376,7 +2483,7 @@ checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2486,8 +2593,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "f72c064e63fbca3138ad07f3588c58093f1684f3a99f60dcfa6d46b87e60fde7"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"crossbeam-epoch 0.8.2",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"fs2",
|
||||
"fxhash",
|
||||
"libc",
|
||||
|
|
@ -2541,7 +2648,7 @@ checksum = "5254766110c377a921c002ca0775d4e384ba69af951fc4329d9dd77af2c25763"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2595,7 +2702,7 @@ dependencies = [
|
|||
"proc-macro-error",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2617,9 +2724,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.45"
|
||||
version = "1.0.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea9c5432ff16d6152371f808fb5a871cd67368171b09bb21b43df8e4a47a3556"
|
||||
checksum = "5ad5de3220ea04da322618ded2c42233d02baca219d6f160a3e9c87cda16c942"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
|
|
@ -2634,7 +2741,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
|
|
@ -2697,7 +2804,7 @@ checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2759,7 +2866,7 @@ checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2802,10 +2909,11 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "tower"
|
||||
version = "0.3.1"
|
||||
source = "git+https://github.com/tower-rs/tower?rev=ad348d8#ad348d8ee5106f21b87155d2a0e8e18b90bd6b73"
|
||||
source = "git+https://github.com/tower-rs/tower?rev=1a84543#1a845433177b31ebc6daff765ee81468c42d6676"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"hdrhistogram",
|
||||
"pin-project",
|
||||
"tokio",
|
||||
"tower-layer",
|
||||
|
|
@ -2846,7 +2954,7 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "tower-layer"
|
||||
version = "0.3.0"
|
||||
source = "git+https://github.com/tower-rs/tower?rev=ad348d8#ad348d8ee5106f21b87155d2a0e8e18b90bd6b73"
|
||||
source = "git+https://github.com/tower-rs/tower?rev=1a84543#1a845433177b31ebc6daff765ee81468c42d6676"
|
||||
|
||||
[[package]]
|
||||
name = "tower-service"
|
||||
|
|
@ -2887,7 +2995,7 @@ checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3073,6 +3181,12 @@ version = "0.8.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.2"
|
||||
|
|
@ -3445,6 +3559,6 @@ checksum = "c3f369ddb18862aba61aa49bf31e74d29f0f162dec753063200e1dc084345d16"
|
|||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.45",
|
||||
"syn 1.0.46",
|
||||
"synstructure",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -21,4 +21,4 @@ panic = "abort"
|
|||
panic = "abort"
|
||||
|
||||
[patch.crates-io]
|
||||
tower = { git = "https://github.com/tower-rs/tower", rev = "ad348d8" }
|
||||
tower = { git = "https://github.com/tower-rs/tower", rev = "1a84543" }
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ rand = "0.7"
|
|||
hyper = "0.13.8"
|
||||
futures = "0.3"
|
||||
tokio = { version = "0.2.22", features = ["time", "rt-threaded", "stream", "macros", "tracing", "signal"] }
|
||||
tower = "0.3"
|
||||
tower = { version = "0.3", features = ["hedge"] }
|
||||
pin-project = "0.4.23"
|
||||
|
||||
color-eyre = { version = "0.5.6", features = ["issue-url"] }
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ use futures::{
|
|||
stream::{FuturesUnordered, StreamExt},
|
||||
};
|
||||
use tokio::time::delay_for;
|
||||
use tower::{builder::ServiceBuilder, retry::Retry, timeout::Timeout, Service, ServiceExt};
|
||||
use tower::{
|
||||
builder::ServiceBuilder, hedge::Hedge, retry::Retry, timeout::Timeout, Service, ServiceExt,
|
||||
};
|
||||
|
||||
use zebra_chain::{
|
||||
block::{self, Block},
|
||||
|
|
@ -16,37 +18,37 @@ use zebra_network as zn;
|
|||
use zebra_state as zs;
|
||||
|
||||
mod downloads;
|
||||
use downloads::Downloads;
|
||||
use downloads::{AlwaysHedge, Downloads};
|
||||
|
||||
/// Controls the number of peers used for each ObtainTips and ExtendTips request.
|
||||
const FANOUT: usize = 4;
|
||||
|
||||
/// Controls how many times we will retry each block download.
|
||||
///
|
||||
/// If all the retries fail, then the syncer will reset, and start downloading
|
||||
/// blocks from the verified tip in the state, including blocks which previously
|
||||
/// downloaded successfully.
|
||||
/// Failing block downloads is important because it defends against peers who
|
||||
/// feed us bad hashes. But spurious failures of valid blocks cause the syncer to
|
||||
/// restart from the previous checkpoint, potentially re-downloading blocks.
|
||||
///
|
||||
/// But if a node is on a slow or unreliable network, sync restarts can result
|
||||
/// in a flood of download requests, making future syncs more likely to fail.
|
||||
/// So it's much faster to retry each block multiple times.
|
||||
///
|
||||
/// When we implement a peer reputation system, we can reduce the number of
|
||||
/// retries, because we will be more likely to choose a good peer.
|
||||
const BLOCK_DOWNLOAD_RETRY_LIMIT: usize = 5;
|
||||
/// We also hedge requests, so we may retry up to twice this many times.
|
||||
const BLOCK_DOWNLOAD_RETRY_LIMIT: usize = 2;
|
||||
|
||||
/// Controls how far ahead of the chain tip the syncer tries to download before
|
||||
/// waiting for queued verifications to complete. Set to twice the maximum
|
||||
/// checkpoint distance.
|
||||
/// waiting for queued verifications to complete.
|
||||
///
|
||||
/// Some checkpoints contain larger blocks, so the maximum checkpoint gap can
|
||||
/// represent multiple gigabytes of data.
|
||||
/// Increasing this limit increases the buffer size, so it reduces the impact of
|
||||
/// missing a block on the critical path. The block size limit is 2MB, so in
|
||||
/// theory, this could represent multiple gigabytes of data, if we downloaded
|
||||
/// arbitrary blocks. However, because we randomly load balance outbound
|
||||
/// requests, and separate block download from obtaining block hashes, an
|
||||
/// adversary would have to control a significant fraction of our peers to lead
|
||||
/// us astray.
|
||||
const LOOKAHEAD_LIMIT: usize = zebra_consensus::MAX_CHECKPOINT_HEIGHT_GAP * 2;
|
||||
|
||||
/// Controls how long we wait for a tips response to return.
|
||||
///
|
||||
/// The network layer also imposes a timeout on requests.
|
||||
const TIPS_RESPONSE_TIMEOUT: Duration = Duration::from_secs(6);
|
||||
|
||||
/// Controls how long we wait for a block download request to complete.
|
||||
///
|
||||
/// The network layer also imposes a timeout on requests.
|
||||
|
|
@ -70,18 +72,6 @@ const BLOCK_VERIFY_TIMEOUT: Duration = Duration::from_secs(MAX_CHECKPOINT_DOWNLO
|
|||
/// Controls how long we wait to restart syncing after finishing a sync run.
|
||||
///
|
||||
/// This timeout should be long enough to:
|
||||
/// - allow pending downloads and verifies to complete or time out.
|
||||
/// Sync restarts don't cancel downloads, so quick restarts can overload
|
||||
/// network-bound nodes with lots of peers, leading to further failures.
|
||||
/// (The total number of requests being processed by peers is the sum of
|
||||
/// the number of peers, and the peer request buffer size.)
|
||||
///
|
||||
/// We assume that Zebra nodes have at least 10 Mbps bandwidth. So a
|
||||
/// maximum-sized block can take up to 2 seconds to download. Therefore, we
|
||||
/// set this timeout to twice the default number of peers. (The peer request
|
||||
/// buffer size is small enough that any buffered requests will overlap with
|
||||
/// the post-restart ObtainTips.)
|
||||
///
|
||||
/// - allow zcashd peers to process pending requests. If the node only has a
|
||||
/// few peers, we want to clear as much peer state as possible. In
|
||||
/// particular, zcashd sends "next block range" hints, based on zcashd's
|
||||
|
|
@ -90,7 +80,7 @@ const BLOCK_VERIFY_TIMEOUT: Duration = Duration::from_secs(MAX_CHECKPOINT_DOWNLO
|
|||
///
|
||||
/// This timeout is particularly important on instances with slow or unreliable
|
||||
/// networks, and on testnet, which has a small number of slow peers.
|
||||
const SYNC_RESTART_TIMEOUT: Duration = Duration::from_secs(100);
|
||||
const SYNC_RESTART_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
|
||||
|
||||
|
|
@ -102,7 +92,6 @@ struct CheckedTip {
|
|||
expected_next: block::Hash,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ChainSync<ZN, ZS, ZV>
|
||||
where
|
||||
ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
|
||||
|
|
@ -118,7 +107,8 @@ where
|
|||
state: ZS,
|
||||
prospective_tips: HashSet<CheckedTip>,
|
||||
genesis_hash: block::Hash,
|
||||
downloads: Pin<Box<Downloads<Retry<zn::RetryLimit, Timeout<ZN>>, Timeout<ZV>>>>,
|
||||
downloads:
|
||||
Pin<Box<Downloads<Hedge<Retry<zn::RetryLimit, Timeout<ZN>>, AlwaysHedge>, Timeout<ZV>>>>,
|
||||
}
|
||||
|
||||
/// Polls the network to determine whether further blocks are available and
|
||||
|
|
@ -143,17 +133,30 @@ where
|
|||
/// - verifier: the zebra-consensus verifier that checks the chain
|
||||
pub fn new(chain: Network, peers: ZN, state: ZS, verifier: ZV) -> Self {
|
||||
let tip_network = Timeout::new(peers.clone(), TIPS_RESPONSE_TIMEOUT);
|
||||
let downloads = Downloads::new(
|
||||
// The Hedge middleware is the outermost layer, hedging requests
|
||||
// between two retry-wrapped networks. The innermost timeout
|
||||
// layer is relatively unimportant, because slow requests will
|
||||
// probably be pre-emptively hedged.
|
||||
//
|
||||
// XXX add ServiceBuilder::hedge() so this becomes
|
||||
// ServiceBuilder::new().hedge(...).retry(...)...
|
||||
let block_network = Hedge::new(
|
||||
ServiceBuilder::new()
|
||||
.retry(zn::RetryLimit::new(BLOCK_DOWNLOAD_RETRY_LIMIT))
|
||||
.timeout(BLOCK_DOWNLOAD_TIMEOUT)
|
||||
.service(peers),
|
||||
Timeout::new(verifier, BLOCK_VERIFY_TIMEOUT),
|
||||
AlwaysHedge,
|
||||
20,
|
||||
0.95,
|
||||
2 * SYNC_RESTART_TIMEOUT,
|
||||
);
|
||||
Self {
|
||||
tip_network,
|
||||
state,
|
||||
downloads: Box::pin(downloads),
|
||||
downloads: Box::pin(Downloads::new(
|
||||
block_network,
|
||||
Timeout::new(verifier, BLOCK_VERIFY_TIMEOUT),
|
||||
)),
|
||||
prospective_tips: HashSet::new(),
|
||||
genesis_hash: genesis_hash(chain),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ use futures::{
|
|||
};
|
||||
use pin_project::pin_project;
|
||||
use tokio::{sync::oneshot, task::JoinHandle};
|
||||
use tower::{Service, ServiceExt};
|
||||
use tower::{hedge, Service, ServiceExt};
|
||||
use tracing_futures::Instrument;
|
||||
|
||||
use zebra_chain::block::{self, Block};
|
||||
|
|
@ -20,6 +20,18 @@ use zebra_network as zn;
|
|||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub(super) struct AlwaysHedge;
|
||||
|
||||
impl<Request: Clone> hedge::Policy<Request> for AlwaysHedge {
|
||||
fn can_retry(&self, _req: &Request) -> bool {
|
||||
true
|
||||
}
|
||||
fn clone_request(&self, req: &Request) -> Option<Request> {
|
||||
Some(req.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a [`Stream`] of download and verification tasks during chain sync.
|
||||
#[pin_project]
|
||||
#[derive(Debug)]
|
||||
|
|
|
|||
Loading…
Reference in New Issue