From 9499d3d2f8e00c951517b492284a1316cab7bd86 Mon Sep 17 00:00:00 2001 From: "Jeff Washington (jwash)" Date: Fri, 18 Aug 2023 08:12:08 -0700 Subject: [PATCH] move `map` to `CacheHashDataFileReference` (#32883) refactor cache hash data map --- accounts-db/src/cache_hash_data.rs | 115 +++++++++++++++-------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/accounts-db/src/cache_hash_data.rs b/accounts-db/src/cache_hash_data.rs index a5516982b2..f25040c92d 100644 --- a/accounts-db/src/cache_hash_data.rs +++ b/accounts-db/src/cache_hash_data.rs @@ -37,6 +37,63 @@ pub(crate) struct CacheHashDataFile { capacity: u64, } +impl CacheHashDataFileReference { + /// convert the open file refrence to a mmapped file that can be returned as a slice + pub(crate) fn map( + &self, + stats: &mut CacheHashDataStats, + ) -> Result { + let file_len = self.file_len; + let mut m1 = Measure::start("read_file"); + let mmap = CacheHashDataFileReference::load_map(&self.file)?; + m1.stop(); + stats.read_us = m1.as_us(); + let header_size = std::mem::size_of::
() as u64; + if file_len < header_size { + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); + } + + let cell_size = std::mem::size_of::() as u64; + unsafe { + assert_eq!( + mmap.align_to::().0.len(), + 0, + "mmap is not aligned" + ); + } + assert_eq!((cell_size as usize) % std::mem::size_of::(), 0); + let mut cache_file = CacheHashDataFile { + mmap, + cell_size, + capacity: 0, + }; + let header = cache_file.get_header_mut(); + let entries = header.count; + + let capacity = cell_size * (entries as u64) + header_size; + if file_len < capacity { + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); + } + cache_file.capacity = capacity; + assert_eq!( + capacity, file_len, + "expected: {capacity}, len on disk: {file_len} {}, entries: {entries}, cell_size: {cell_size}", self.path.display(), + ); + + stats.total_entries = entries; + stats.cache_file_size += capacity as usize; + + stats.loaded_from_cache += 1; + stats.entries_loaded_from_cache += entries; + + Ok(cache_file) + } + + fn load_map(file: &File) -> Result { + Ok(unsafe { MmapMut::map_mut(file).unwrap() }) + } +} + impl CacheHashDataFile { /// return a slice of a reference to all the cache hash data from the mmapped file pub fn get_cache_hash_data(&self) -> &[EntryType] { @@ -136,10 +193,6 @@ impl CacheHashDataFile { data.flush().unwrap(); Ok(unsafe { MmapMut::map_mut(&data).unwrap() }) } - - fn load_map(file: &File) -> Result { - Ok(unsafe { MmapMut::map_mut(file).unwrap() }) - } } pub type PreExistingCacheFiles = HashSet; @@ -250,63 +303,11 @@ impl CacheHashData { ) -> Result { let mut stats = CacheHashDataStats::default(); let reference = self.get_file_reference_to_map_later(file_name)?; - let result = self.map(reference, &mut stats); + let result = reference.map(&mut stats); self.stats.lock().unwrap().accumulate(&stats); result } - /// create and return a MappedCacheFile for a cache file path - fn map( - &self, - reference: CacheHashDataFileReference, - stats: &mut CacheHashDataStats, - ) -> Result { - let file_len = reference.file_len; - let mut m1 = Measure::start("read_file"); - let mmap = CacheHashDataFile::load_map(&reference.file)?; - m1.stop(); - stats.read_us = m1.as_us(); - let header_size = std::mem::size_of::
() as u64; - if file_len < header_size { - return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); - } - - let cell_size = std::mem::size_of::() as u64; - unsafe { - assert_eq!( - mmap.align_to::().0.len(), - 0, - "mmap is not aligned" - ); - } - assert_eq!((cell_size as usize) % std::mem::size_of::(), 0); - let mut cache_file = CacheHashDataFile { - mmap, - cell_size, - capacity: 0, - }; - let header = cache_file.get_header_mut(); - let entries = header.count; - - let capacity = cell_size * (entries as u64) + header_size; - if file_len < capacity { - return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); - } - cache_file.capacity = capacity; - assert_eq!( - capacity, file_len, - "expected: {capacity}, len on disk: {file_len} {}, entries: {entries}, cell_size: {cell_size}", reference.path.display(), - ); - - stats.total_entries = entries; - stats.cache_file_size += capacity as usize; - - stats.loaded_from_cache += 1; - stats.entries_loaded_from_cache += entries; - - Ok(cache_file) - } - pub(crate) fn pre_existing_cache_file_will_be_used(&self, file_name: impl AsRef) { self.pre_existing_cache_files .lock()