2021-07-15 17:40:07 -07:00
//! Persistent storage for accounts.
//!
//! For more information, see:
//!
//! <https://docs.solana.com/implemented-proposals/persistent-account-storage>
2021-03-02 06:36:49 -08:00
2021-12-03 09:00:31 -08:00
use {
log ::* ,
memmap2 ::MmapMut ,
serde ::{ Deserialize , Serialize } ,
solana_sdk ::{
account ::{ Account , AccountSharedData , ReadableAccount } ,
clock ::{ Epoch , Slot } ,
hash ::Hash ,
pubkey ::Pubkey ,
} ,
std ::{
borrow ::Borrow ,
convert ::TryFrom ,
fs ::{ remove_file , OpenOptions } ,
io ::{ self , Seek , SeekFrom , Write } ,
mem ,
path ::{ Path , PathBuf } ,
sync ::{
atomic ::{ AtomicUsize , Ordering } ,
Mutex ,
} ,
} ,
2019-09-23 15:20:45 -07:00
} ;
2019-01-31 18:37:05 -08:00
2022-05-20 11:37:45 -07:00
pub mod test_utils ;
2021-03-03 12:51:48 -08:00
// Data placement should be aligned at the next boundary. Without alignment accessing the memory may
// crash on some architectures.
2021-12-15 13:41:11 -08:00
pub const ALIGN_BOUNDARY_OFFSET : usize = mem ::size_of ::< u64 > ( ) ;
2019-12-17 21:10:36 -08:00
macro_rules ! u64_align {
( $addr : expr ) = > {
( $addr + ( ALIGN_BOUNDARY_OFFSET - 1 ) ) & ! ( ALIGN_BOUNDARY_OFFSET - 1 )
2019-03-04 22:36:12 -08:00
} ;
}
2021-12-15 13:41:11 -08:00
pub const MAXIMUM_APPEND_VEC_FILE_SIZE : u64 = 16 * 1024 * 1024 * 1024 ; // 16 GiB
2020-01-06 15:14:56 -08:00
2021-05-17 16:58:36 -07:00
pub type StoredMetaWriteVersion = u64 ;
2019-09-23 15:20:45 -07:00
/// Meta contains enough context to recover the index from storage itself
2019-12-11 20:03:33 -08:00
/// This struct will be backed by mmaped and snapshotted data files.
/// So the data layout must be stable and consistent across the entire cluster!
2022-05-22 18:00:42 -07:00
#[ derive(Clone, PartialEq, Eq, Debug) ]
2019-09-23 15:20:45 -07:00
pub struct StoredMeta {
2019-04-15 17:15:50 -07:00
/// global write version
2021-05-17 16:58:36 -07:00
pub write_version : StoredMetaWriteVersion ,
2019-04-15 17:15:50 -07:00
/// key for the account
pub pubkey : Pubkey ,
2019-04-16 08:50:05 -07:00
pub data_len : u64 ,
}
2019-12-11 20:03:33 -08:00
/// This struct will be backed by mmaped and snapshotted data files.
/// So the data layout must be stable and consistent across the entire cluster!
2019-05-30 21:31:35 -07:00
#[ derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq) ]
2019-09-23 15:20:45 -07:00
pub struct AccountMeta {
2019-04-16 08:50:05 -07:00
/// lamports in the account
pub lamports : u64 ,
/// the program that owns this account. If executable, the program that loads this account.
pub owner : Pubkey ,
/// this account's data contains a loaded program (and is now read-only)
pub executable : bool ,
2019-08-23 14:04:53 -07:00
/// the epoch at which this account will next owe rent
pub rent_epoch : Epoch ,
2019-04-16 08:50:05 -07:00
}
2021-04-21 10:20:37 -07:00
impl < ' a , T : ReadableAccount > From < & ' a T > for AccountMeta {
fn from ( account : & ' a T ) -> Self {
2021-01-11 17:00:23 -08:00
Self {
2021-04-21 10:20:37 -07:00
lamports : account . lamports ( ) ,
owner : * account . owner ( ) ,
executable : account . executable ( ) ,
rent_epoch : account . rent_epoch ( ) ,
2021-01-11 17:00:23 -08:00
}
}
}
2021-04-28 06:47:26 -07:00
impl < ' a , T : ReadableAccount > From < Option < & ' a T > > for AccountMeta {
fn from ( account : Option < & ' a T > ) -> Self {
match account {
Some ( account ) = > AccountMeta ::from ( account ) ,
None = > AccountMeta ::default ( ) ,
}
}
}
2021-03-03 12:51:48 -08:00
/// References to account data stored elsewhere. Getting an `Account` requires cloning
/// (see `StoredAccountMeta::clone_account()`).
2022-05-22 18:00:42 -07:00
#[ derive(PartialEq, Eq, Debug) ]
2021-01-11 17:00:23 -08:00
pub struct StoredAccountMeta < ' a > {
2019-09-23 15:20:45 -07:00
pub meta : & ' a StoredMeta ,
2019-04-15 17:15:50 -07:00
/// account data
2019-09-23 15:20:45 -07:00
pub account_meta : & ' a AccountMeta ,
2019-04-16 08:50:05 -07:00
pub data : & ' a [ u8 ] ,
2019-05-30 21:31:35 -07:00
pub offset : usize ,
2021-01-11 17:00:23 -08:00
pub stored_size : usize ,
2019-09-20 13:21:12 -07:00
pub hash : & ' a Hash ,
2019-04-16 08:50:05 -07:00
}
2021-01-11 17:00:23 -08:00
impl < ' a > StoredAccountMeta < ' a > {
2021-03-03 12:51:48 -08:00
/// Return a new Account by copying all the data referenced by the `StoredAccountMeta`.
2021-03-09 13:06:07 -08:00
pub fn clone_account ( & self ) -> AccountSharedData {
2021-03-11 16:09:04 -08:00
AccountSharedData ::from ( Account {
2019-09-23 15:20:45 -07:00
lamports : self . account_meta . lamports ,
owner : self . account_meta . owner ,
executable : self . account_meta . executable ,
rent_epoch : self . account_meta . rent_epoch ,
2019-04-16 08:50:05 -07:00
data : self . data . to_vec ( ) ,
2021-03-11 16:09:04 -08:00
} )
2019-04-16 08:50:05 -07:00
}
2019-12-17 21:10:36 -08:00
2022-10-31 13:58:42 -07:00
pub fn pubkey ( & self ) -> & Pubkey {
& self . meta . pubkey
}
2019-12-17 21:10:36 -08:00
fn sanitize ( & self ) -> bool {
2020-03-28 23:45:45 -07:00
self . sanitize_executable ( ) & & self . sanitize_lamports ( )
}
fn sanitize_executable ( & self ) -> bool {
2019-12-17 21:10:36 -08:00
// Sanitize executable to ensure higher 7-bits are cleared correctly.
self . ref_executable_byte ( ) & ! 1 = = 0
}
2020-03-28 23:45:45 -07:00
fn sanitize_lamports ( & self ) -> bool {
2021-03-09 13:06:07 -08:00
// Sanitize 0 lamports to ensure to be same as AccountSharedData::default()
self . account_meta . lamports ! = 0 | | self . clone_account ( ) = = AccountSharedData ::default ( )
2020-03-28 23:45:45 -07:00
}
2019-12-17 21:10:36 -08:00
fn ref_executable_byte ( & self ) -> & u8 {
// Use extra references to avoid value silently clamped to 1 (=true) and 0 (=false)
2021-01-28 08:15:33 -08:00
// Yes, this really happens; see test_new_from_file_crafted_executable
2019-12-17 21:10:36 -08:00
let executable_bool : & bool = & self . account_meta . executable ;
// UNSAFE: Force to interpret mmap-backed bool as u8 to really read the actual memory content
let executable_byte : & u8 = unsafe { & * ( executable_bool as * const bool as * const u8 ) } ;
executable_byte
}
2019-04-15 17:15:50 -07:00
}
2022-05-20 06:50:24 -07:00
pub struct AppendVecAccountsIter < ' a > {
append_vec : & ' a AppendVec ,
offset : usize ,
}
impl < ' a > AppendVecAccountsIter < ' a > {
pub fn new ( append_vec : & ' a AppendVec ) -> Self {
Self {
append_vec ,
offset : 0 ,
}
}
}
impl < ' a > Iterator for AppendVecAccountsIter < ' a > {
type Item = StoredAccountMeta < ' a > ;
fn next ( & mut self ) -> Option < Self ::Item > {
if let Some ( ( account , next_offset ) ) = self . append_vec . get_account ( self . offset ) {
self . offset = next_offset ;
Some ( account )
} else {
None
}
}
}
2021-03-03 12:51:48 -08:00
/// A thread-safe, file-backed block of memory used to store `Account` instances. Append operations
/// are serialized such that only one thread updates the internal `append_lock` at a time. No
/// restrictions are placed on reading. That is, one may read items from one thread while another
/// is appending new items.
2020-07-06 04:22:23 -07:00
#[ derive(Debug, AbiExample) ]
2019-04-12 04:30:17 -07:00
pub struct AppendVec {
2021-03-03 12:51:48 -08:00
/// The file path where the data is stored.
2019-05-30 21:31:35 -07:00
path : PathBuf ,
2021-03-03 12:51:48 -08:00
/// A file-backed block of memory that is used to store the data for each appended item.
2019-04-12 04:30:17 -07:00
map : MmapMut ,
2021-03-03 12:51:48 -08:00
/// A lock used to serialize append operations.
append_lock : Mutex < ( ) > ,
/// The number of bytes used to store items, not the number of items.
2019-04-12 04:30:17 -07:00
current_len : AtomicUsize ,
2021-03-03 12:51:48 -08:00
/// The number of bytes available for storing items.
2019-01-31 18:37:05 -08:00
file_size : u64 ,
2021-03-03 12:51:48 -08:00
/// True if the file should automatically be deleted when this AppendVec is dropped.
2020-11-01 09:41:28 -08:00
remove_on_drop : bool ,
2019-02-27 11:23:26 -08:00
}
2019-05-30 21:31:35 -07:00
impl Drop for AppendVec {
fn drop ( & mut self ) {
2020-11-01 09:41:28 -08:00
if self . remove_on_drop {
2021-02-02 06:52:30 -08:00
if let Err ( _e ) = remove_file ( & self . path ) {
2020-12-28 11:59:09 -08:00
// promote this to panic soon.
2021-02-02 06:52:30 -08:00
// disabled due to many false positive warnings while running tests.
2021-03-02 06:36:49 -08:00
// blocked by rpc's upgrade to jsonrpc v17
2021-02-02 06:52:30 -08:00
//error!("AppendVec failed to remove {:?}: {:?}", &self.path, e);
2022-07-05 09:27:01 -07:00
inc_new_counter_info! ( " append_vec_drop_fail " , 1 ) ;
2020-12-28 11:59:09 -08:00
}
2020-11-01 09:41:28 -08:00
}
2019-05-30 21:31:35 -07:00
}
}
2019-04-12 04:30:17 -07:00
impl AppendVec {
pub fn new ( file : & Path , create : bool , size : usize ) -> Self {
2020-01-06 15:14:56 -08:00
let initial_len = 0 ;
AppendVec ::sanitize_len_and_size ( initial_len , size ) . unwrap ( ) ;
2019-07-23 13:47:48 -07:00
if create {
let _ignored = remove_file ( file ) ;
}
2019-01-31 18:37:05 -08:00
let mut data = OpenOptions ::new ( )
. read ( true )
. write ( true )
. create ( create )
2019-04-12 04:30:17 -07:00
. open ( file )
2019-07-08 15:37:54 -07:00
. map_err ( | e | {
panic! (
2020-12-28 11:59:09 -08:00
" Unable to {} data file {} in current dir({:?}): {:?} " ,
2019-07-08 15:37:54 -07:00
if create { " create " } else { " open " } ,
file . display ( ) ,
2020-12-28 11:59:09 -08:00
std ::env ::current_dir ( ) ,
2019-07-08 15:37:54 -07:00
e
) ;
} )
. unwrap ( ) ;
2019-01-31 18:37:05 -08:00
2021-03-02 06:36:49 -08:00
// Theoretical performance optimization: write a zero to the end of
// the file so that we won't have to resize it later, which may be
// expensive.
2019-05-01 09:27:13 -07:00
data . seek ( SeekFrom ::Start ( ( size - 1 ) as u64 ) ) . unwrap ( ) ;
2019-01-31 18:37:05 -08:00
data . write_all ( & [ 0 ] ) . unwrap ( ) ;
data . seek ( SeekFrom ::Start ( 0 ) ) . unwrap ( ) ;
data . flush ( ) . unwrap ( ) ;
2021-03-02 06:36:49 -08:00
2019-04-16 10:53:37 -07:00
//UNSAFE: Required to create a Mmap
2020-04-06 01:30:23 -07:00
let map = unsafe { MmapMut ::map_mut ( & data ) } ;
2020-09-15 15:43:17 -07:00
let map = map . unwrap_or_else ( | e | {
error! (
" Failed to map the data file (size: {}): {}. \n
Please increase sysctl vm . max_map_count or equivalent for your platform . " ,
size , e
) ;
std ::process ::exit ( 1 ) ;
} ) ;
2019-01-31 18:37:05 -08:00
AppendVec {
2019-05-30 21:31:35 -07:00
path : file . to_path_buf ( ) ,
2019-04-12 04:30:17 -07:00
map ,
// This mutex forces append to be single threaded, but concurrent with reads
2019-04-16 10:53:37 -07:00
// See UNSAFE usage in `append_ptr`
2021-03-03 12:51:48 -08:00
append_lock : Mutex ::new ( ( ) ) ,
2020-01-06 15:14:56 -08:00
current_len : AtomicUsize ::new ( initial_len ) ,
2019-04-12 04:30:17 -07:00
file_size : size as u64 ,
2020-11-01 09:41:28 -08:00
remove_on_drop : true ,
2019-01-31 18:37:05 -08:00
}
}
2020-11-01 09:41:28 -08:00
pub fn set_no_remove_on_drop ( & mut self ) {
self . remove_on_drop = false ;
}
2020-01-06 15:14:56 -08:00
fn sanitize_len_and_size ( current_len : usize , file_size : usize ) -> io ::Result < ( ) > {
if file_size = = 0 {
Err ( std ::io ::Error ::new (
std ::io ::ErrorKind ::Other ,
format! ( " too small file size {} for AppendVec " , file_size ) ,
) )
2021-12-01 23:26:03 -08:00
} else if usize ::try_from ( MAXIMUM_APPEND_VEC_FILE_SIZE )
. map ( | max | file_size > max )
. unwrap_or ( true )
{
2020-01-06 15:14:56 -08:00
Err ( std ::io ::Error ::new (
std ::io ::ErrorKind ::Other ,
format! ( " too large file size {} for AppendVec " , file_size ) ,
) )
} else if current_len > file_size {
Err ( std ::io ::Error ::new (
std ::io ::ErrorKind ::Other ,
format! ( " current_len is larger than file size ( {} ) " , file_size ) ,
) )
} else {
Ok ( ( ) )
}
}
2019-09-25 18:07:41 -07:00
pub fn flush ( & self ) -> io ::Result < ( ) > {
self . map . flush ( )
}
2019-04-12 04:30:17 -07:00
pub fn reset ( & self ) {
// This mutex forces append to be single threaded, but concurrent with reads
2019-04-16 10:53:37 -07:00
// See UNSAFE usage in `append_ptr`
2021-03-03 12:51:48 -08:00
let _lock = self . append_lock . lock ( ) . unwrap ( ) ;
2022-02-20 18:30:49 -08:00
self . current_len . store ( 0 , Ordering ::Release ) ;
2019-01-31 18:37:05 -08:00
}
2022-04-20 11:44:57 -07:00
/// how many more bytes can be stored in this append vec
pub fn remaining_bytes ( & self ) -> u64 {
( self . capacity ( ) ) . saturating_sub ( self . len ( ) as u64 )
}
2019-04-12 04:30:17 -07:00
pub fn len ( & self ) -> usize {
2022-02-20 18:30:49 -08:00
self . current_len . load ( Ordering ::Acquire )
2019-04-11 13:16:56 -07:00
}
2019-04-12 04:30:17 -07:00
pub fn is_empty ( & self ) -> bool {
self . len ( ) = = 0
}
pub fn capacity ( & self ) -> u64 {
self . file_size
}
2021-12-11 19:38:13 -08:00
pub fn file_name ( slot : Slot , id : impl std ::fmt ::Display ) -> String {
2021-03-10 11:26:37 -08:00
format! ( " {} . {} " , slot , id )
2019-08-05 22:53:19 -07:00
}
2021-01-28 08:15:33 -08:00
pub fn new_from_file < P : AsRef < Path > > ( path : P , current_len : usize ) -> io ::Result < ( Self , usize ) > {
2022-07-28 12:57:26 -07:00
let new = Self ::new_from_file_unchecked ( path , current_len ) ? ;
let ( sanitized , num_accounts ) = new . sanitize_layout_and_length ( ) ;
if ! sanitized {
return Err ( std ::io ::Error ::new (
std ::io ::ErrorKind ::Other ,
" incorrect layout/length/data " ,
) ) ;
}
Ok ( ( new , num_accounts ) )
}
/// Creates an appendvec from file without performing sanitize checks or counting the number of accounts
pub fn new_from_file_unchecked < P : AsRef < Path > > (
path : P ,
current_len : usize ,
) -> io ::Result < Self > {
let file_size = std ::fs ::metadata ( & path ) ? . len ( ) ;
Self ::sanitize_len_and_size ( current_len , file_size as usize ) ? ;
2019-08-05 22:53:19 -07:00
let data = OpenOptions ::new ( )
. read ( true )
. write ( true )
. create ( false )
. open ( & path ) ? ;
2021-06-03 20:22:12 -07:00
let map = unsafe {
let result = MmapMut ::map_mut ( & data ) ;
if result . is_err ( ) {
// for vm.max_map_count, error is: {code: 12, kind: Other, message: "Cannot allocate memory"}
info! ( " memory map error: {:?}. This may be because vm.max_map_count is not set correctly. " , result ) ;
}
result ?
} ;
2020-01-06 15:14:56 -08:00
2022-07-28 12:57:26 -07:00
Ok ( AppendVec {
2021-01-28 08:15:33 -08:00
path : path . as_ref ( ) . to_path_buf ( ) ,
map ,
2021-03-03 12:51:48 -08:00
append_lock : Mutex ::new ( ( ) ) ,
2021-01-28 08:15:33 -08:00
current_len : AtomicUsize ::new ( current_len ) ,
file_size ,
remove_on_drop : true ,
2022-07-28 12:57:26 -07:00
} )
2019-08-05 22:53:19 -07:00
}
2021-01-23 08:05:05 -08:00
fn sanitize_layout_and_length ( & self ) -> ( bool , usize ) {
2019-12-17 21:10:36 -08:00
let mut offset = 0 ;
// This discards allocated accounts immediately after check at each loop iteration.
//
// This code should not reuse AppendVec.accounts() method as the current form or
// extend it to be reused here because it would allow attackers to accumulate
// some measurable amount of memory needlessly.
2021-01-23 08:05:05 -08:00
let mut num_accounts = 0 ;
2019-12-17 21:10:36 -08:00
while let Some ( ( account , next_offset ) ) = self . get_account ( offset ) {
if ! account . sanitize ( ) {
2021-01-23 08:05:05 -08:00
return ( false , num_accounts ) ;
2019-12-17 21:10:36 -08:00
}
offset = next_offset ;
2021-01-23 08:05:05 -08:00
num_accounts + = 1 ;
2019-12-17 21:10:36 -08:00
}
2022-02-20 18:30:49 -08:00
let aligned_current_len = u64_align! ( self . current_len . load ( Ordering ::Acquire ) ) ;
2019-12-17 21:10:36 -08:00
2021-01-23 08:05:05 -08:00
( offset = = aligned_current_len , num_accounts )
2019-12-17 21:10:36 -08:00
}
2021-03-03 12:51:48 -08:00
/// Get a reference to the data at `offset` of `size` bytes if that slice
/// doesn't overrun the internal buffer. Otherwise return None.
/// Also return the offset of the first byte after the requested data that
/// falls on a 64-byte boundary.
2019-04-16 08:50:05 -07:00
fn get_slice ( & self , offset : usize , size : usize ) -> Option < ( & [ u8 ] , usize ) > {
2019-12-17 21:10:36 -08:00
let ( next , overflow ) = offset . overflowing_add ( size ) ;
if overflow | | next > self . len ( ) {
2019-04-16 08:50:05 -07:00
return None ;
2019-01-31 18:37:05 -08:00
}
2019-12-17 21:10:36 -08:00
let data = & self . map [ offset .. next ] ;
let next = u64_align! ( next ) ;
2019-04-16 08:50:05 -07:00
Some ( (
2019-04-16 10:53:37 -07:00
//UNSAFE: This unsafe creates a slice that represents a chunk of self.map memory
//The lifetime of this slice is tied to &self, since it points to self.map memory
2019-04-16 08:50:05 -07:00
unsafe { std ::slice ::from_raw_parts ( data . as_ptr ( ) as * const u8 , size ) } ,
next ,
) )
2019-01-31 18:37:05 -08:00
}
2021-03-03 12:51:48 -08:00
/// Copy `len` bytes from `src` to the first 64-byte boundary after position `offset` of
/// the internal buffer. Then update `offset` to the first byte after the copied data.
2019-04-12 04:30:17 -07:00
fn append_ptr ( & self , offset : & mut usize , src : * const u8 , len : usize ) {
2019-12-17 21:10:36 -08:00
let pos = u64_align! ( * offset ) ;
2019-04-12 04:30:17 -07:00
let data = & self . map [ pos .. ( pos + len ) ] ;
2019-04-16 10:53:37 -07:00
//UNSAFE: This mut append is safe because only 1 thread can append at a time
2021-03-03 12:51:48 -08:00
//Mutex<()> guarantees exclusive write access to the memory occupied in
2019-04-16 10:53:37 -07:00
//the range.
2019-04-12 04:30:17 -07:00
unsafe {
let dst = data . as_ptr ( ) as * mut u8 ;
std ::ptr ::copy ( src , dst , len ) ;
} ;
* offset = pos + len ;
}
2021-03-03 12:51:48 -08:00
/// Copy each value in `vals`, in order, to the first 64-byte boundary after position `offset`.
/// If there is sufficient space, then update `offset` and the internal `current_len` to the
/// first byte after the copied data and return the starting position of the copied data.
/// Otherwise return None and leave `offset` unchanged.
2019-04-24 09:51:57 -07:00
fn append_ptrs_locked ( & self , offset : & mut usize , vals : & [ ( * const u8 , usize ) ] ) -> Option < usize > {
2019-04-12 04:30:17 -07:00
let mut end = * offset ;
for val in vals {
2019-12-17 21:10:36 -08:00
end = u64_align! ( end ) ;
2019-04-12 04:30:17 -07:00
end + = val . 1 ;
}
2019-01-31 18:37:05 -08:00
2019-05-01 09:27:13 -07:00
if ( self . file_size as usize ) < end {
2019-01-31 18:37:05 -08:00
return None ;
}
2019-12-17 21:10:36 -08:00
let pos = u64_align! ( * offset ) ;
2019-04-12 04:30:17 -07:00
for val in vals {
2019-04-24 09:51:57 -07:00
self . append_ptr ( offset , val . 0 , val . 1 )
2019-04-12 04:30:17 -07:00
}
2022-02-20 18:30:49 -08:00
self . current_len . store ( * offset , Ordering ::Release ) ;
2019-04-12 04:30:17 -07:00
Some ( pos )
}
2021-03-03 12:51:48 -08:00
/// Return a reference to the type at `offset` if its data doesn't overrun the internal buffer.
/// Otherwise return None. Also return the offset of the first byte after the requested data
/// that falls on a 64-byte boundary.
2019-04-16 10:53:37 -07:00
fn get_type < ' a , T > ( & self , offset : usize ) -> Option < ( & ' a T , usize ) > {
2019-04-16 08:50:05 -07:00
let ( data , next ) = self . get_slice ( offset , mem ::size_of ::< T > ( ) ) ? ;
let ptr : * const T = data . as_ptr ( ) as * const T ;
2019-04-16 10:53:37 -07:00
//UNSAFE: The cast is safe because the slice is aligned and fits into the memory
2021-03-03 12:51:48 -08:00
//and the lifetime of the &T is tied to self, which holds the underlying memory map
2019-04-16 08:50:05 -07:00
Some ( ( unsafe { & * ptr } , next ) )
}
2021-03-03 12:51:48 -08:00
/// Return account metadata for the account at `offset` if its data doesn't overrun
/// the internal buffer. Otherwise return None. Also return the offset of the first byte
/// after the requested data that falls on a 64-byte boundary.
2021-01-11 17:00:23 -08:00
pub fn get_account < ' a > ( & ' a self , offset : usize ) -> Option < ( StoredAccountMeta < ' a > , usize ) > {
2019-09-23 15:20:45 -07:00
let ( meta , next ) : ( & ' a StoredMeta , _ ) = self . get_type ( offset ) ? ;
let ( account_meta , next ) : ( & ' a AccountMeta , _ ) = self . get_type ( next ) ? ;
2019-09-20 13:21:12 -07:00
let ( hash , next ) : ( & ' a Hash , _ ) = self . get_type ( next ) ? ;
2019-04-16 08:50:05 -07:00
let ( data , next ) = self . get_slice ( next , meta . data_len as usize ) ? ;
2021-01-11 17:00:23 -08:00
let stored_size = next - offset ;
2019-04-16 08:50:05 -07:00
Some ( (
2021-01-11 17:00:23 -08:00
StoredAccountMeta {
2019-04-16 08:50:05 -07:00
meta ,
2019-09-23 15:20:45 -07:00
account_meta ,
2019-04-16 08:50:05 -07:00
data ,
2019-05-30 21:31:35 -07:00
offset ,
2021-01-11 17:00:23 -08:00
stored_size ,
2019-09-20 13:21:12 -07:00
hash ,
2019-04-16 08:50:05 -07:00
} ,
next ,
) )
}
2022-04-07 07:05:20 -07:00
#[ cfg(test) ]
2021-03-09 13:06:07 -08:00
pub fn get_account_test ( & self , offset : usize ) -> Option < ( StoredMeta , AccountSharedData ) > {
2019-09-23 15:20:45 -07:00
let ( stored_account , _ ) = self . get_account ( offset ) ? ;
let meta = stored_account . meta . clone ( ) ;
Some ( ( meta , stored_account . clone_account ( ) ) )
2019-01-31 18:37:05 -08:00
}
2019-07-31 17:58:10 -07:00
pub fn get_path ( & self ) -> PathBuf {
self . path . clone ( )
}
2022-07-12 16:56:05 -07:00
/// Return iterator for account metadata
pub fn account_iter ( & self ) -> AppendVecAccountsIter {
AppendVecAccountsIter ::new ( self )
}
/// Return a vector of account metadata for each account, starting from `offset`.
2021-03-03 12:51:48 -08:00
pub fn accounts ( & self , mut offset : usize ) -> Vec < StoredAccountMeta > {
2019-04-12 04:30:17 -07:00
let mut accounts = vec! [ ] ;
2021-03-03 12:51:48 -08:00
while let Some ( ( account , next ) ) = self . get_account ( offset ) {
2019-04-16 08:50:05 -07:00
accounts . push ( account ) ;
2021-03-03 12:51:48 -08:00
offset = next ;
2019-04-12 04:30:17 -07:00
}
accounts
2019-01-31 18:37:05 -08:00
}
2021-03-03 12:51:48 -08:00
/// Copy each account metadata, account and hash to the internal buffer.
2022-11-16 10:59:11 -08:00
/// If there is no room to write the first entry, None is returned.
/// Otherwise, returns the starting offset of each account metadata.
/// Plus, the final return value is the offset where the next entry would be appended.
/// So, return.len() is 1 + (number of accounts written)
2021-03-03 12:51:48 -08:00
/// After each account is appended, the internal `current_len` is updated
/// and will be available to other threads.
2019-09-20 13:21:12 -07:00
pub fn append_accounts (
& self ,
2021-04-28 13:29:22 -07:00
accounts : & [ ( StoredMeta , Option < & impl ReadableAccount > ) ] ,
2021-04-27 07:10:06 -07:00
hashes : & [ impl Borrow < Hash > ] ,
2022-11-16 10:59:11 -08:00
) -> Option < Vec < usize > > {
2021-03-03 12:51:48 -08:00
let _lock = self . append_lock . lock ( ) . unwrap ( ) ;
let mut offset = self . len ( ) ;
2019-10-30 21:55:17 -07:00
let mut rv = Vec ::with_capacity ( accounts . len ( ) ) ;
2019-09-23 15:20:45 -07:00
for ( ( stored_meta , account ) , hash ) in accounts . iter ( ) . zip ( hashes ) {
let meta_ptr = stored_meta as * const StoredMeta ;
2021-01-11 17:00:23 -08:00
let account_meta = AccountMeta ::from ( * account ) ;
2019-09-23 15:20:45 -07:00
let account_meta_ptr = & account_meta as * const AccountMeta ;
let data_len = stored_meta . data_len as usize ;
2021-04-28 06:47:26 -07:00
let data_ptr = account
. map ( | account | account . data ( ) )
. unwrap_or_default ( )
. as_ptr ( ) ;
2021-04-27 07:10:06 -07:00
let hash_ptr = hash . borrow ( ) . as_ref ( ) . as_ptr ( ) ;
2019-04-24 09:51:57 -07:00
let ptrs = [
2019-09-23 15:20:45 -07:00
( meta_ptr as * const u8 , mem ::size_of ::< StoredMeta > ( ) ) ,
( account_meta_ptr as * const u8 , mem ::size_of ::< AccountMeta > ( ) ) ,
2019-09-20 13:21:12 -07:00
( hash_ptr as * const u8 , mem ::size_of ::< Hash > ( ) ) ,
2019-04-24 09:51:57 -07:00
( data_ptr , data_len ) ,
] ;
if let Some ( res ) = self . append_ptrs_locked ( & mut offset , & ptrs ) {
rv . push ( res )
} else {
break ;
}
}
2021-01-11 17:00:23 -08:00
2022-11-16 10:59:11 -08:00
if rv . is_empty ( ) {
None
} else {
// The last entry in this offset needs to be the u64 aligned offset, because that's
// where the *next* entry will begin to be stored.
rv . push ( u64_align! ( offset ) ) ;
2021-01-11 17:00:23 -08:00
2022-11-16 10:59:11 -08:00
Some ( rv )
}
2019-04-24 09:51:57 -07:00
}
2019-04-12 04:30:17 -07:00
}
2019-04-11 13:16:56 -07:00
2019-01-31 18:37:05 -08:00
#[ cfg(test) ]
pub mod tests {
2021-12-03 09:00:31 -08:00
use {
super ::{ test_utils ::* , * } ,
assert_matches ::assert_matches ,
rand ::{ thread_rng , Rng } ,
solana_sdk ::{ account ::WritableAccount , timing ::duration_as_ms } ,
std ::time ::Instant ,
} ;
2019-01-31 18:37:05 -08:00
2019-09-23 15:20:45 -07:00
impl AppendVec {
2021-03-09 13:06:07 -08:00
fn append_account_test ( & self , data : & ( StoredMeta , AccountSharedData ) ) -> Option < usize > {
2022-11-17 12:26:51 -08:00
self . append_accounts (
& [ ( data . 0. clone ( ) , Some ( & data . 1. clone ( ) ) ) ] ,
& [ Hash ::default ( ) ] ,
)
. map ( | res | res [ 0 ] )
2019-09-23 15:20:45 -07:00
}
}
2021-01-11 17:00:23 -08:00
impl < ' a > StoredAccountMeta < ' a > {
2020-05-15 09:35:43 -07:00
#[ allow(clippy::cast_ref_to_mut) ]
2019-12-17 21:10:36 -08:00
fn set_data_len_unsafe ( & self , new_data_len : u64 ) {
// UNSAFE: cast away & (= const ref) to &mut to force to mutate append-only (=read-only) AppendVec
2020-05-15 09:35:43 -07:00
unsafe {
* ( & self . meta . data_len as * const u64 as * mut u64 ) = new_data_len ;
}
2019-12-17 21:10:36 -08:00
}
fn get_executable_byte ( & self ) -> u8 {
let executable_bool : bool = self . account_meta . executable ;
// UNSAFE: Force to interpret mmap-backed bool as u8 to really read the actual memory content
let executable_byte : u8 = unsafe { std ::mem ::transmute ::< bool , u8 > ( executable_bool ) } ;
executable_byte
}
2020-05-15 09:35:43 -07:00
#[ allow(clippy::cast_ref_to_mut) ]
2019-12-17 21:10:36 -08:00
fn set_executable_as_byte ( & self , new_executable_byte : u8 ) {
// UNSAFE: Force to interpret mmap-backed &bool as &u8 to write some crafted value;
2020-05-15 09:35:43 -07:00
unsafe {
* ( & self . account_meta . executable as * const bool as * mut u8 ) = new_executable_byte ;
}
2019-12-17 21:10:36 -08:00
}
}
2021-04-28 06:47:26 -07:00
#[ test ]
fn test_account_meta_default ( ) {
let def1 = AccountMeta ::default ( ) ;
let def2 = AccountMeta ::from ( & Account ::default ( ) ) ;
assert_eq! ( & def1 , & def2 ) ;
let def2 = AccountMeta ::from ( & AccountSharedData ::default ( ) ) ;
assert_eq! ( & def1 , & def2 ) ;
let def2 = AccountMeta ::from ( Some ( & AccountSharedData ::default ( ) ) ) ;
assert_eq! ( & def1 , & def2 ) ;
let none : Option < & AccountSharedData > = None ;
let def2 = AccountMeta ::from ( none ) ;
assert_eq! ( & def1 , & def2 ) ;
}
#[ test ]
fn test_account_meta_non_default ( ) {
let def1 = AccountMeta {
lamports : 1 ,
owner : Pubkey ::new_unique ( ) ,
executable : true ,
rent_epoch : 3 ,
} ;
let def2_account = Account {
lamports : def1 . lamports ,
owner : def1 . owner ,
executable : def1 . executable ,
rent_epoch : def1 . rent_epoch ,
data : Vec ::new ( ) ,
} ;
let def2 = AccountMeta ::from ( & def2_account ) ;
assert_eq! ( & def1 , & def2 ) ;
let def2 = AccountMeta ::from ( & AccountSharedData ::from ( def2_account . clone ( ) ) ) ;
assert_eq! ( & def1 , & def2 ) ;
let def2 = AccountMeta ::from ( Some ( & AccountSharedData ::from ( def2_account ) ) ) ;
assert_eq! ( & def1 , & def2 ) ;
}
2020-01-06 15:14:56 -08:00
#[ test ]
#[ should_panic(expected = " too small file size 0 for AppendVec " ) ]
fn test_append_vec_new_bad_size ( ) {
let path = get_append_vec_path ( " test_append_vec_new_bad_size " ) ;
let _av = AppendVec ::new ( & path . path , true , 0 ) ;
}
#[ test ]
2021-01-28 08:15:33 -08:00
fn test_append_vec_new_from_file_bad_size ( ) {
let file = get_append_vec_path ( " test_append_vec_new_from_file_bad_size " ) ;
2020-01-06 15:14:56 -08:00
let path = & file . path ;
let _data = OpenOptions ::new ( )
. read ( true )
. write ( true )
. create ( true )
2022-09-22 15:23:03 -07:00
. open ( path )
2020-01-06 15:14:56 -08:00
. expect ( " create a test file for mmap " ) ;
2021-01-28 08:15:33 -08:00
let result = AppendVec ::new_from_file ( path , 0 ) ;
2020-01-06 15:14:56 -08:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " too small file size 0 for AppendVec " ) ;
}
#[ test ]
fn test_append_vec_sanitize_len_and_size_too_small ( ) {
2020-05-15 09:35:43 -07:00
const LEN : usize = 0 ;
const SIZE : usize = 0 ;
let result = AppendVec ::sanitize_len_and_size ( LEN , SIZE ) ;
2020-01-06 15:14:56 -08:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " too small file size 0 for AppendVec " ) ;
}
#[ test ]
fn test_append_vec_sanitize_len_and_size_maximum ( ) {
2020-05-15 09:35:43 -07:00
const LEN : usize = 0 ;
const SIZE : usize = 16 * 1024 * 1024 * 1024 ;
let result = AppendVec ::sanitize_len_and_size ( LEN , SIZE ) ;
2020-01-06 15:14:56 -08:00
assert_matches! ( result , Ok ( _ ) ) ;
}
#[ test ]
fn test_append_vec_sanitize_len_and_size_too_large ( ) {
2020-05-15 09:35:43 -07:00
const LEN : usize = 0 ;
const SIZE : usize = 16 * 1024 * 1024 * 1024 + 1 ;
let result = AppendVec ::sanitize_len_and_size ( LEN , SIZE ) ;
2020-01-06 15:14:56 -08:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " too large file size 17179869185 for AppendVec " ) ;
}
#[ test ]
fn test_append_vec_sanitize_len_and_size_full_and_same_as_current_len ( ) {
2020-05-15 09:35:43 -07:00
const LEN : usize = 1024 * 1024 ;
const SIZE : usize = 1024 * 1024 ;
let result = AppendVec ::sanitize_len_and_size ( LEN , SIZE ) ;
2020-01-06 15:14:56 -08:00
assert_matches! ( result , Ok ( _ ) ) ;
}
#[ test ]
fn test_append_vec_sanitize_len_and_size_larger_current_len ( ) {
2020-05-15 09:35:43 -07:00
const LEN : usize = 1024 * 1024 + 1 ;
const SIZE : usize = 1024 * 1024 ;
let result = AppendVec ::sanitize_len_and_size ( LEN , SIZE ) ;
2020-01-06 15:14:56 -08:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " current_len is larger than file size (1048576) " ) ;
}
2019-01-31 18:37:05 -08:00
#[ test ]
2019-04-16 08:50:05 -07:00
fn test_append_vec_one ( ) {
2019-04-12 04:30:17 -07:00
let path = get_append_vec_path ( " test_append " ) ;
let av = AppendVec ::new ( & path . path , true , 1024 * 1024 ) ;
let account = create_test_account ( 0 ) ;
2019-04-16 08:50:05 -07:00
let index = av . append_account_test ( & account ) . unwrap ( ) ;
assert_eq! ( av . get_account_test ( index ) . unwrap ( ) , account ) ;
2019-01-31 18:37:05 -08:00
}
2022-04-20 11:44:57 -07:00
#[ test ]
fn test_remaining_bytes ( ) {
let path = get_append_vec_path ( " test_append " ) ;
let sz = 1024 * 1024 ;
let sz64 = sz as u64 ;
let av = AppendVec ::new ( & path . path , true , sz ) ;
assert_eq! ( av . capacity ( ) , sz64 ) ;
assert_eq! ( av . remaining_bytes ( ) , sz64 ) ;
let account = create_test_account ( 0 ) ;
let acct_size = 136 ;
av . append_account_test ( & account ) . unwrap ( ) ;
assert_eq! ( av . capacity ( ) , sz64 ) ;
assert_eq! ( av . remaining_bytes ( ) , sz64 - acct_size ) ;
}
2019-01-31 18:37:05 -08:00
#[ test ]
2019-04-12 04:30:17 -07:00
fn test_append_vec_data ( ) {
let path = get_append_vec_path ( " test_append_data " ) ;
let av = AppendVec ::new ( & path . path , true , 1024 * 1024 ) ;
let account = create_test_account ( 5 ) ;
2019-04-16 08:50:05 -07:00
let index = av . append_account_test ( & account ) . unwrap ( ) ;
assert_eq! ( av . get_account_test ( index ) . unwrap ( ) , account ) ;
2019-04-12 04:30:17 -07:00
let account1 = create_test_account ( 6 ) ;
2019-04-16 08:50:05 -07:00
let index1 = av . append_account_test ( & account1 ) . unwrap ( ) ;
assert_eq! ( av . get_account_test ( index ) . unwrap ( ) , account ) ;
assert_eq! ( av . get_account_test ( index1 ) . unwrap ( ) , account1 ) ;
2019-01-31 18:37:05 -08:00
}
#[ test ]
2019-04-12 04:30:17 -07:00
fn test_append_vec_append_many ( ) {
let path = get_append_vec_path ( " test_append_many " ) ;
let av = AppendVec ::new ( & path . path , true , 1024 * 1024 ) ;
let size = 1000 ;
let mut indexes = vec! [ ] ;
let now = Instant ::now ( ) ;
for sample in 0 .. size {
let account = create_test_account ( sample ) ;
2019-04-16 08:50:05 -07:00
let pos = av . append_account_test ( & account ) . unwrap ( ) ;
assert_eq! ( av . get_account_test ( pos ) . unwrap ( ) , account ) ;
2019-04-12 04:30:17 -07:00
indexes . push ( pos )
}
trace! ( " append time: {} ms " , duration_as_ms ( & now . elapsed ( ) ) , ) ;
2019-01-31 18:37:05 -08:00
let now = Instant ::now ( ) ;
2019-04-12 04:30:17 -07:00
for _ in 0 .. size {
let sample = thread_rng ( ) . gen_range ( 0 , indexes . len ( ) ) ;
let account = create_test_account ( sample ) ;
2019-04-16 08:50:05 -07:00
assert_eq! ( av . get_account_test ( indexes [ sample ] ) . unwrap ( ) , account ) ;
2019-01-31 18:37:05 -08:00
}
2019-04-12 04:30:17 -07:00
trace! ( " random read time: {} ms " , duration_as_ms ( & now . elapsed ( ) ) , ) ;
2019-01-31 18:37:05 -08:00
let now = Instant ::now ( ) ;
2019-04-12 04:30:17 -07:00
assert_eq! ( indexes . len ( ) , size ) ;
assert_eq! ( indexes [ 0 ] , 0 ) ;
2019-04-16 08:50:05 -07:00
let mut accounts = av . accounts ( indexes [ 0 ] ) ;
2019-04-12 04:30:17 -07:00
assert_eq! ( accounts . len ( ) , size ) ;
2019-04-16 08:50:05 -07:00
for ( sample , v ) in accounts . iter_mut ( ) . enumerate ( ) {
2019-04-12 04:30:17 -07:00
let account = create_test_account ( sample ) ;
2019-04-16 08:50:05 -07:00
let recovered = v . clone_account ( ) ;
assert_eq! ( recovered , account . 1 )
2019-01-31 18:37:05 -08:00
}
2019-04-12 04:30:17 -07:00
trace! (
" sequential read time: {} ms " ,
2019-01-31 18:37:05 -08:00
duration_as_ms ( & now . elapsed ( ) ) ,
) ;
}
2019-05-30 21:31:35 -07:00
2020-03-28 23:45:45 -07:00
#[ test ]
2021-01-28 08:15:33 -08:00
fn test_new_from_file_crafted_zero_lamport_account ( ) {
2022-11-21 12:43:03 -08:00
// This test verifies that when we sanitize on load, that we fail sanitizing if we load an account with zero lamports that does not have all default value fields.
// This test writes an account with zero lamports, but with 3 bytes of data. On load, it asserts that load fails.
// It used to be possible to use the append vec api to write an account to an append vec with zero lamports, but with non-default values for other account fields.
// This will no longer be possible. Thus, to implement the write portion of this test would require additional test-only parameters to public apis or otherwise duplicating code paths.
// So, the sanitizing on load behavior can be tested by capturing [u8] that would be created if such a write was possible (as it used to be).
// The contents of [u8] written by an append vec cannot easily or reasonably change frequently since it has released a long time.
/*
solana_logger ::setup ( ) ;
// uncomment this code to generate the invalid append vec that will fail on load
let file = get_append_vec_path ( " test_append " ) ;
let path = & file . path ;
let mut av = AppendVec ::new ( path , true , 256 ) ;
av . set_no_remove_on_drop ( ) ;
let pubkey = solana_sdk ::pubkey ::new_rand ( ) ;
let owner = Pubkey ::default ( ) ;
let data_len = 3_ u64 ;
let mut account = AccountSharedData ::new ( 0 , data_len as usize , & owner ) ;
account . set_data ( b " abc " . to_vec ( ) ) ;
let stored_meta = StoredMeta {
write_version : 0 ,
pubkey ,
data_len ,
} ;
let account_with_meta = ( stored_meta , account ) ;
let index = av . append_account_test ( & account_with_meta ) . unwrap ( ) ;
assert_eq! ( av . get_account_test ( index ) . unwrap ( ) , account_with_meta ) ;
av . flush ( ) . unwrap ( ) ;
let accounts_len = av . len ( ) ;
drop ( av ) ;
// read file and log out as [u8]
use std ::fs ::File ;
use std ::io ::BufReader ;
use std ::io ::Read ;
let f = File ::open ( path ) . unwrap ( ) ;
let mut reader = BufReader ::new ( f ) ;
let mut buffer = Vec ::new ( ) ;
reader . read_to_end ( & mut buffer ) . unwrap ( ) ;
error! ( " {:?} " , buffer ) ;
* /
// create an invalid append vec file using known bytes
let file = get_append_vec_path ( " test_append_bytes " ) ;
2020-03-28 23:45:45 -07:00
let path = & file . path ;
2022-11-21 12:43:03 -08:00
let accounts_len = 139 ;
{
let append_vec_data = [
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 192 , 118 , 150 , 1 , 185 , 209 , 118 ,
82 , 154 , 222 , 172 , 202 , 110 , 26 , 218 , 140 , 143 , 96 , 61 , 43 , 212 , 73 , 203 , 7 , 190 ,
88 , 80 , 222 , 110 , 114 , 67 , 254 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 97 , 98 , 99 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
] ;
let f = std ::fs ::File ::create ( path ) . unwrap ( ) ;
let mut writer = std ::io ::BufWriter ::new ( f ) ;
writer . write_all ( append_vec_data . as_slice ( ) ) . unwrap ( ) ;
}
2020-03-28 23:45:45 -07:00
2021-01-28 08:15:33 -08:00
let result = AppendVec ::new_from_file ( path , accounts_len ) ;
2020-03-28 23:45:45 -07:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " incorrect layout/length/data " ) ;
}
2019-12-17 21:10:36 -08:00
#[ test ]
2021-01-28 08:15:33 -08:00
fn test_new_from_file_crafted_data_len ( ) {
let file = get_append_vec_path ( " test_new_from_file_crafted_data_len " ) ;
2019-12-17 21:10:36 -08:00
let path = & file . path ;
2021-06-18 06:34:46 -07:00
let mut av = AppendVec ::new ( path , true , 1024 * 1024 ) ;
2021-01-28 08:15:33 -08:00
av . set_no_remove_on_drop ( ) ;
2019-12-17 21:10:36 -08:00
let crafted_data_len = 1 ;
av . append_account_test ( & create_test_account ( 10 ) ) . unwrap ( ) ;
let accounts = av . accounts ( 0 ) ;
let account = accounts . first ( ) . unwrap ( ) ;
account . set_data_len_unsafe ( crafted_data_len ) ;
assert_eq! ( account . meta . data_len , crafted_data_len ) ;
2020-06-17 20:54:52 -07:00
// Reload accounts and observe crafted_data_len
2019-12-17 21:10:36 -08:00
let accounts = av . accounts ( 0 ) ;
let account = accounts . first ( ) . unwrap ( ) ;
assert_eq! ( account . meta . data_len , crafted_data_len ) ;
av . flush ( ) . unwrap ( ) ;
2021-01-28 08:15:33 -08:00
let accounts_len = av . len ( ) ;
drop ( av ) ;
let result = AppendVec ::new_from_file ( path , accounts_len ) ;
2020-03-28 23:45:45 -07:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " incorrect layout/length/data " ) ;
2019-12-17 21:10:36 -08:00
}
#[ test ]
2021-01-28 08:15:33 -08:00
fn test_new_from_file_too_large_data_len ( ) {
let file = get_append_vec_path ( " test_new_from_file_too_large_data_len " ) ;
2019-12-17 21:10:36 -08:00
let path = & file . path ;
2021-06-18 06:34:46 -07:00
let mut av = AppendVec ::new ( path , true , 1024 * 1024 ) ;
2021-01-28 08:15:33 -08:00
av . set_no_remove_on_drop ( ) ;
2019-12-17 21:10:36 -08:00
let too_large_data_len = u64 ::max_value ( ) ;
av . append_account_test ( & create_test_account ( 10 ) ) . unwrap ( ) ;
let accounts = av . accounts ( 0 ) ;
let account = accounts . first ( ) . unwrap ( ) ;
account . set_data_len_unsafe ( too_large_data_len ) ;
assert_eq! ( account . meta . data_len , too_large_data_len ) ;
// Reload accounts and observe no account with bad offset
let accounts = av . accounts ( 0 ) ;
assert_matches! ( accounts . first ( ) , None ) ;
av . flush ( ) . unwrap ( ) ;
2021-01-28 08:15:33 -08:00
let accounts_len = av . len ( ) ;
drop ( av ) ;
let result = AppendVec ::new_from_file ( path , accounts_len ) ;
2020-03-28 23:45:45 -07:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " incorrect layout/length/data " ) ;
2019-12-17 21:10:36 -08:00
}
#[ test ]
2021-01-28 08:15:33 -08:00
fn test_new_from_file_crafted_executable ( ) {
let file = get_append_vec_path ( " test_new_from_crafted_executable " ) ;
2019-12-17 21:10:36 -08:00
let path = & file . path ;
2021-06-18 06:34:46 -07:00
let mut av = AppendVec ::new ( path , true , 1024 * 1024 ) ;
2021-01-28 08:15:33 -08:00
av . set_no_remove_on_drop ( ) ;
2019-12-17 21:10:36 -08:00
av . append_account_test ( & create_test_account ( 10 ) ) . unwrap ( ) ;
{
let mut executable_account = create_test_account ( 10 ) ;
2021-04-28 07:07:43 -07:00
executable_account . 1. set_executable ( true ) ;
2019-12-17 21:10:36 -08:00
av . append_account_test ( & executable_account ) . unwrap ( ) ;
}
// reload accounts
let accounts = av . accounts ( 0 ) ;
// ensure false is 0u8 and true is 1u8 actually
assert_eq! ( * accounts [ 0 ] . ref_executable_byte ( ) , 0 ) ;
assert_eq! ( * accounts [ 1 ] . ref_executable_byte ( ) , 1 ) ;
let account = & accounts [ 0 ] ;
let crafted_executable = u8 ::max_value ( ) - 1 ;
account . set_executable_as_byte ( crafted_executable ) ;
// reload crafted accounts
let accounts = av . accounts ( 0 ) ;
let account = accounts . first ( ) . unwrap ( ) ;
2022-06-27 10:03:52 -07:00
// upper 7-bits are not 0, so sanitization should fail
assert! ( ! account . sanitize_executable ( ) ) ;
2019-12-17 21:10:36 -08:00
// we can observe crafted value by ref
{
let executable_bool : & bool = & account . account_meta . executable ;
// Depending on use, *executable_bool can be truthy or falsy due to direct memory manipulation
2020-06-17 20:54:52 -07:00
// assert_eq! thinks *executable_bool is equal to false but the if condition thinks it's not, contradictorily.
2021-05-19 07:31:47 -07:00
assert! ( ! * executable_bool ) ;
2022-06-27 10:03:52 -07:00
#[ cfg(not(target_arch = " aarch64 " )) ]
{
const FALSE : bool = false ; // keep clippy happy
if * executable_bool = = FALSE {
panic! ( " This didn't occur if this test passed. " ) ;
}
2019-12-17 21:10:36 -08:00
}
assert_eq! ( * account . ref_executable_byte ( ) , crafted_executable ) ;
}
// we can NOT observe crafted value by value
{
let executable_bool : bool = account . account_meta . executable ;
2021-05-19 07:31:47 -07:00
assert! ( ! executable_bool ) ;
2019-12-17 21:10:36 -08:00
assert_eq! ( account . get_executable_byte ( ) , 0 ) ; // Wow, not crafted_executable!
}
av . flush ( ) . unwrap ( ) ;
2021-01-28 08:15:33 -08:00
let accounts_len = av . len ( ) ;
drop ( av ) ;
let result = AppendVec ::new_from_file ( path , accounts_len ) ;
2020-03-28 23:45:45 -07:00
assert_matches! ( result , Err ( ref message ) if message . to_string ( ) = = * " incorrect layout/length/data " ) ;
2019-12-17 21:10:36 -08:00
}
2019-01-31 18:37:05 -08:00
}