From 8ee0e9632c1c2686de55011a6bd18fe71eca59ac Mon Sep 17 00:00:00 2001 From: Sathish <44555499+sambley@users.noreply.github.com> Date: Fri, 14 Dec 2018 15:10:10 -0800 Subject: [PATCH] Switch to using hashbrown version of HashMap and (#2158) HashSet for improved performance and memory usage --- Cargo.lock | 11 +++++++++++ Cargo.toml | 1 + src/bank.rs | 5 +++-- src/cluster_info.rs | 2 +- src/crds_gossip_pull.rs | 2 +- src/crds_gossip_push.rs | 2 +- src/leader_scheduler.rs | 4 ++-- src/lib.rs | 1 + src/thin_client.rs | 2 +- 9 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 554eea72e..5c5933944 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -654,6 +654,15 @@ name = "half" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "hashbrown" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hex-literal" version = "0.1.1" @@ -1732,6 +1741,7 @@ dependencies = [ "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2798,6 +2808,7 @@ dependencies = [ "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum h2 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "7dd33bafe2e6370e6c8eb0cf1b8c5f93390b90acde7e9b03723f166b28b648ed" "checksum half 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9353c2a89d550b58fa0061d8ed8d002a7d8cdf2494eb0e432859bd3a9e543836" +"checksum hashbrown 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "64b7d419d0622ae02fe5da6b9a5e1964b610a65bb37923b976aeebb6dbb8f86e" "checksum hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4da5f0e01bd8a71a224a4eedecaacfcabda388dbb7a80faf04d3514287572d95" "checksum hex-literal-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d340b6514f232f6db1bd16db65302a5278a04fef9ce867cb932e7e5fa21130a" "checksum http 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "02096a6d2c55e63f7fcb800690e4f889a25f6ec342e3adb4594e293b625215ab" diff --git a/Cargo.toml b/Cargo.toml index 5afe1a7f6..c1089bb9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ solana-vote-signer = { path = "vote-signer", version = "0.0.1" } tokio = "0.1" tokio-codec = "0.1" untrusted = "0.6.2" +hashbrown = "0.1.7" [[bench]] name = "bank" diff --git a/src/bank.rs b/src/bank.rs index 4323e402d..71e4f3196 100644 --- a/src/bank.rs +++ b/src/bank.rs @@ -16,6 +16,7 @@ use crate::runtime::{self, RuntimeError}; use crate::storage_stage::StorageState; use bincode::deserialize; use bincode::serialize; +use hashbrown::{HashMap, HashSet}; use itertools::Itertools; use log::Level; use rayon::prelude::*; @@ -38,7 +39,7 @@ use solana_sdk::token_program; use solana_sdk::transaction::Transaction; use solana_sdk::vote_program; use std; -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, VecDeque}; use std::result; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, RwLock}; @@ -207,7 +208,7 @@ pub struct Accounts { impl Accounts { /// Returns a read-only iterator over all known accounts - pub fn account_values(&self) -> std::collections::hash_map::Values { + pub fn account_values(&self) -> hashbrown::hash_map::Values { self.accounts.values() } diff --git a/src/cluster_info.rs b/src/cluster_info.rs index d545042dc..a31ff2469 100644 --- a/src/cluster_info.rs +++ b/src/cluster_info.rs @@ -27,6 +27,7 @@ use crate::rpc::RPC_PORT; use crate::streamer::{BlobReceiver, BlobSender}; use crate::window::{SharedWindow, WindowIndex}; use bincode::{deserialize, serialize}; +use hashbrown::HashMap; use log::Level; use rand::{thread_rng, Rng}; use rayon::prelude::*; @@ -35,7 +36,6 @@ use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil, Signable, Signature}; use solana_sdk::timing::{duration_as_ms, timestamp}; -use std::collections::HashMap; use std::io; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/src/crds_gossip_pull.rs b/src/crds_gossip_pull.rs index a7739b3d8..ee1e71a78 100644 --- a/src/crds_gossip_pull.rs +++ b/src/crds_gossip_pull.rs @@ -16,12 +16,12 @@ use crate::crds_gossip_error::CrdsGossipError; use crate::crds_value::{CrdsValue, CrdsValueLabel}; use crate::packet::BLOB_DATA_SIZE; use bincode::serialized_size; +use hashbrown::HashMap; use rand; use rand::distributions::{Distribution, WeightedIndex}; use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use std::cmp; -use std::collections::HashMap; use std::collections::VecDeque; pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000; diff --git a/src/crds_gossip_push.rs b/src/crds_gossip_push.rs index d27180ca6..60a5963fa 100644 --- a/src/crds_gossip_push.rs +++ b/src/crds_gossip_push.rs @@ -16,13 +16,13 @@ use crate::crds_gossip_error::CrdsGossipError; use crate::crds_value::{CrdsValue, CrdsValueLabel}; use crate::packet::BLOB_DATA_SIZE; use bincode::serialized_size; +use hashbrown::HashMap; use indexmap::map::IndexMap; use rand; use rand::seq::SliceRandom; use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use std::cmp; -use std::collections::HashMap; pub const CRDS_GOSSIP_NUM_ACTIVE: usize = 30; pub const CRDS_GOSSIP_PUSH_FANOUT: usize = 6; diff --git a/src/leader_scheduler.rs b/src/leader_scheduler.rs index 39cf8f930..8f7edd06a 100644 --- a/src/leader_scheduler.rs +++ b/src/leader_scheduler.rs @@ -7,6 +7,7 @@ use crate::entry::Entry; use crate::ledger::create_ticks; use bincode::serialize; use byteorder::{LittleEndian, ReadBytesExt}; +use hashbrown::HashSet; use solana_sdk::hash::{hash, Hash}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; @@ -14,7 +15,6 @@ use solana_sdk::system_transaction::SystemTransaction; use solana_sdk::transaction::Transaction; use solana_sdk::vote_program::{self, Vote, VoteProgram}; use solana_sdk::vote_transaction::VoteTransaction; -use std::collections::HashSet; use std::io::Cursor; pub const DEFAULT_BOOTSTRAP_HEIGHT: u64 = 1000; @@ -519,13 +519,13 @@ mod tests { DEFAULT_LEADER_ROTATION_INTERVAL, DEFAULT_SEED_ROTATION_INTERVAL, }; use crate::mint::Mint; + use hashbrown::HashSet; use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::transaction::Transaction; use solana_sdk::vote_program::Vote; use solana_sdk::vote_transaction::VoteTransaction; - use std::collections::HashSet; use std::hash::Hash as StdHash; use std::iter::FromIterator; diff --git a/src/lib.rs b/src/lib.rs index b2816aa7a..bb688e3e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -77,6 +77,7 @@ pub mod window_service; #[cfg(any(feature = "chacha", feature = "cuda"))] #[macro_use] extern crate hex_literal; +extern crate hashbrown; #[macro_use] extern crate log; diff --git a/src/thin_client.rs b/src/thin_client.rs index d54fd9194..caf0ab039 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -11,6 +11,7 @@ use crate::result::{Error, Result}; use crate::rpc_request::{RpcClient, RpcRequest}; use bincode::serialize; use bs58; +use hashbrown::HashMap; use log::Level; use serde_json; use solana_metrics; @@ -23,7 +24,6 @@ use solana_sdk::system_transaction::SystemTransaction; use solana_sdk::timing; use solana_sdk::transaction::Transaction; use std; -use std::collections::HashMap; use std::io; use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::AtomicBool;