Add GenesisBlock::OperatingMode to control how cluster features are activated (#6430)

Cargo.lock

@@ -3460,6 +3460,7 @@ dependencies = [
 name = "solana-genesis-programs"
 version = "0.20.0"
 dependencies = [
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-bpf-loader-api 0.20.0",
  "solana-bpf-loader-program 0.20.0",
  "solana-budget-api 0.20.0",
@@ -3470,6 +3471,7 @@ dependencies = [
  "solana-exchange-program 0.20.0",
  "solana-move-loader-api 0.20.0",
  "solana-move-loader-program 0.20.0",
+ "solana-runtime 0.20.0",
  "solana-sdk 0.20.0",
  "solana-stake-api 0.20.0",
  "solana-stake-program 0.20.0",
@@ -3561,6 +3563,7 @@ dependencies = [
  "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-budget-api 0.20.0",
+ "solana-genesis-programs 0.20.0",
  "solana-logger 0.20.0",
  "solana-measure 0.20.0",
  "solana-merkle-tree 0.20.0",
@@ -3618,10 +3621,12 @@ dependencies = [
  "solana-bench-exchange 0.20.0",
  "solana-bench-tps 0.20.0",
  "solana-client 0.20.0",
+ "solana-config-api 0.20.0",
  "solana-core 0.20.0",
  "solana-drone 0.20.0",
  "solana-exchange-api 0.20.0",
  "solana-exchange-program 0.20.0",
+ "solana-genesis-programs 0.20.0",
  "solana-ledger 0.20.0",
  "solana-logger 0.20.0",
  "solana-move-loader-api 0.20.0",
@@ -3632,6 +3637,7 @@ dependencies = [
  "solana-stake-api 0.20.0",
  "solana-storage-api 0.20.0",
  "solana-storage-program 0.20.0",
+ "solana-vest-api 0.20.0",
  "solana-vote-api 0.20.0",
  "symlink 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",

core/src/validator.rs

@@ -393,6 +393,12 @@ fn get_bank_forks(
     verify_ledger: bool,
     dev_halt_at_slot: Option<Slot>,
 ) -> (BankForks, Vec<BankForksInfo>, LeaderScheduleCache) {
+    let process_options = blocktree_processor::ProcessOptions {
+        verify_ledger,
+        dev_halt_at_slot,
+        ..blocktree_processor::ProcessOptions::default()
+    };
+
     if let Some(snapshot_config) = snapshot_config.as_ref() {
         info!(
             "Initializing snapshot path: {:?}",
@@ -417,13 +423,10 @@ fn get_bank_forks(
             .expect("Load from snapshot failed");
 
             return blocktree_processor::process_blocktree_from_root(
+                genesis_block,
                 blocktree,
                 Arc::new(deserialized_bank),
-                &blocktree_processor::ProcessOptions {
+                &process_options,
-                    verify_ledger,
-                    dev_halt_at_slot,
-                    ..blocktree_processor::ProcessOptions::default()
-                },
             )
             .expect("processing blocktree after loading snapshot failed");
         } else {
@@ -438,11 +441,7 @@ fn get_bank_forks(
         &genesis_block,
         &blocktree,
         account_paths,
-        blocktree_processor::ProcessOptions {
+        process_options,
-            verify_ledger,
-            dev_halt_at_slot,
-            ..blocktree_processor::ProcessOptions::default()
-        },
     )
     .expect("process_blocktree failed")
 }

genesis/src/main.rs

@@ -9,7 +9,7 @@ use solana_sdk::{
     clock,
     epoch_schedule::EpochSchedule,
     fee_calculator::FeeCalculator,
-    genesis_block::GenesisBlock,
+    genesis_block::{GenesisBlock, OperatingMode},
     native_token::sol_to_lamports,
     poh_config::PohConfig,
     pubkey::Pubkey,
@@ -285,6 +285,11 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 .takes_value(true)
                 .help("The location of keypairs for primordial accounts and balance"),
         )
+        .arg(
+            Arg::with_name("development")
+                .long("dev")
+                .help("Configure the cluster for development mode where all features are available at epoch 0"),
+        )
         .get_matches();
 
     let bootstrap_leader_keypair_file = matches.value_of("bootstrap_leader_keypair_file").unwrap();
@@ -376,14 +381,21 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         }
     }
 
+    let operating_mode = if matches.is_present("development") {
+        OperatingMode::Development
+    } else {
+        OperatingMode::SoftLaunch
+    };
+    let native_instruction_processors = solana_genesis_programs::get(operating_mode, 0).unwrap();
     let mut genesis_block = GenesisBlock {
         accounts,
-        native_instruction_processors: solana_genesis_programs::get(),
+        native_instruction_processors,
         ticks_per_slot,
         epoch_schedule,
         fee_calculator,
         rent_calculator,
         poh_config,
+        operating_mode,
         ..GenesisBlock::default()
     };
 

genesis_programs/Cargo.toml

@@ -9,6 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
+log = { version = "0.4.8" }
 solana-bpf-loader-api = { path = "../programs/bpf_loader_api", version = "0.20.0" }
 solana-bpf-loader-program = { path = "../programs/bpf_loader_program", version = "0.20.0" }
 solana-budget-api = { path = "../programs/budget_api", version = "0.20.0" }
@@ -19,6 +20,7 @@ solana-exchange-api = { path = "../programs/exchange_api", version = "0.20.0" }
 solana-exchange-program = { path = "../programs/exchange_program", version = "0.20.0" }
 solana-move-loader-program = { path = "../programs/move_loader_program", version = "0.20.0", optional = true }
 solana-move-loader-api = { path = "../programs/move_loader_api", version = "0.20.0", optional = true }
+solana-runtime = { path = "../runtime", version = "0.20.0" }
 solana-sdk = { path = "../sdk", version = "0.20.0" }
 solana-stake-api = { path = "../programs/stake_api", version = "0.20.0" }
 solana-stake-program = { path = "../programs/stake_program", version = "0.20.0" }

genesis_programs/src/lib.rs

@@ -1,5 +1,7 @@
-use solana_sdk::pubkey::Pubkey;
+use solana_sdk::{
-use solana_sdk::system_program::solana_system_program;
+    clock::Epoch, genesis_block::OperatingMode, pubkey::Pubkey,
+    system_program::solana_system_program,
+};
 
 #[macro_use]
 extern crate solana_bpf_loader_program;
@@ -21,43 +23,102 @@ extern crate solana_vest_program;
 #[macro_use]
 extern crate solana_vote_program;
 
-pub fn get() -> Vec<(String, Pubkey)> {
+use log::*;
-    vec![
+use solana_runtime::bank::{Bank, EnteredEpochCallback};
-        solana_system_program(),
+
-        solana_bpf_loader_program!(),
+pub fn get(operating_mode: OperatingMode, epoch: Epoch) -> Option<Vec<(String, Pubkey)>> {
-        solana_budget_program!(),
+    match operating_mode {
-        solana_config_program!(),
+        OperatingMode::Development => {
-        solana_exchange_program!(),
+            if epoch == 0 {
-        #[cfg(feature = "move")]
+                Some(vec![
-        solana_move_loader_program!(),
+                    // Enable all SoftLaunch programs
-        solana_stake_program!(),
+                    solana_system_program(),
-        solana_storage_program!(),
+                    solana_bpf_loader_program!(),
-        solana_vest_program!(),
+                    solana_config_program!(),
-        solana_vote_program!(),
+                    solana_stake_program!(),
-    ]
+                    solana_storage_program!(),
+                    solana_vest_program!(),
+                    solana_vote_program!(),
+                    // Programs that are only available in Development mode
+                    solana_budget_program!(),
+                    solana_exchange_program!(),
+                    #[cfg(feature = "move")]
+                    solana_move_loader_program!(),
+                ])
+            } else {
+                None
+            }
+        }
+        OperatingMode::SoftLaunch => {
+            if epoch == 0 {
+                // Voting and Staking only at epoch 0
+                Some(vec![solana_stake_program!(), solana_vote_program!()])
+            } else if epoch == std::u64::MAX - 1 {
+                // System program and Archivers are activated next
+                //
+                // The epoch of std::u64::MAX - 1 is a placeholder and is expected to be reduced in
+                // a future hard fork.
+                Some(vec![
+                    solana_config_program!(),
+                    solana_storage_program!(),
+                    solana_system_program(),
+                    solana_vest_program!(),
+                ])
+            } else if epoch == std::u64::MAX {
+                // Finally 3rd party BPF programs are available
+                //
+                // The epoch of std::u64::MAX is a placeholder and is expected to be reduced in a
+                // future hard fork.
+                Some(vec![solana_bpf_loader_program!()])
+            } else {
+                None
+            }
+        }
+    }
+}
+
+pub fn get_entered_epoch_callback(operating_mode: OperatingMode) -> EnteredEpochCallback {
+    Box::new(move |bank: &mut Bank| {
+        info!(
+            "Entering epoch {} with operating_mode {:?}",
+            bank.epoch(),
+            operating_mode
+        );
+        if let Some(new_programs) = get(operating_mode, bank.epoch()) {
+            for (name, program_id) in new_programs.iter() {
+                info!("Registering {} at {}", name, program_id);
+                bank.register_native_instruction_processor(name, program_id);
+            }
+        }
+    })
 }
 
 #[cfg(test)]
 mod tests {
+    use super::*;
     use std::collections::HashSet;
 
     #[test]
     fn test_id_uniqueness() {
         let mut unique = HashSet::new();
-        let ids = vec![
+        let ids = get(OperatingMode::Development, 0).unwrap();
-            solana_budget_api::id(),
-            solana_config_api::id(),
-            solana_exchange_api::id(),
-            #[cfg(feature = "move")]
-            solana_move_loader_api::id(),
-            solana_sdk::bpf_loader::id(),
-            solana_sdk::native_loader::id(),
-            solana_sdk::system_program::id(),
-            solana_stake_api::id(),
-            solana_storage_api::id(),
-            solana_vest_api::id(),
-            solana_vote_api::id(),
-        ];
         assert!(ids.into_iter().all(move |id| unique.insert(id)));
     }
+
+    #[test]
+    fn test_development_programs() {
+        assert_eq!(get(OperatingMode::Development, 0).unwrap().len(), 9);
+        assert_eq!(get(OperatingMode::Development, 1), None);
+    }
+
+    #[test]
+    fn test_softlaunch_programs() {
+        assert_eq!(
+            get(OperatingMode::SoftLaunch, 0),
+            Some(vec![solana_stake_program!(), solana_vote_program!(),])
+        );
+        assert_eq!(get(OperatingMode::SoftLaunch, 1), None);
+        assert!(get(OperatingMode::SoftLaunch, std::u64::MAX - 1).is_some());
+        assert!(get(OperatingMode::SoftLaunch, std::u64::MAX).is_some());
+    }
 }

ledger/Cargo.toml

@@ -26,6 +26,7 @@ rayon = "1.2.0"
 reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
 serde = "1.0.101"
 serde_derive = "1.0.101"
+solana-genesis-programs = { path = "../genesis_programs", version = "0.20.0" }
 solana-logger = { path = "../logger", version = "0.20.0" }
 solana-measure = { path = "../measure", version = "0.20.0" }
 solana-merkle-tree = { path = "../merkle-tree", version = "0.20.0" }

ledger/src/blocktree_processor.rs

@@ -206,13 +206,14 @@ pub fn process_blocktree(
 
     // Setup bank for slot 0
     let bank0 = Arc::new(Bank::new_with_paths(&genesis_block, account_paths));
-    info!("processing ledger from bank 0...");
+    info!("processing ledger for bank 0...");
     process_bank_0(&bank0, blocktree, &opts)?;
-    process_blocktree_from_root(blocktree, bank0, &opts)
+    process_blocktree_from_root(genesis_block, blocktree, bank0, &opts)
 }
 
 // Process blocktree from a known root bank
 pub fn process_blocktree_from_root(
+    genesis_block: &GenesisBlock,
     blocktree: &Blocktree,
     bank: Arc<Bank>,
     opts: &ProcessOptions,
@@ -224,6 +225,10 @@ pub fn process_blocktree_from_root(
     let now = Instant::now();
     let mut rooted_path = vec![start_slot];
 
+    bank.set_entered_epoch_callback(solana_genesis_programs::get_entered_epoch_callback(
+        genesis_block.operating_mode,
+    ));
+
     blocktree
         .set_roots(&[start_slot])
         .expect("Couldn't set root on startup");
@@ -1699,7 +1704,7 @@ pub mod tests {
 
         // Test process_blocktree_from_root() from slot 1 onwards
         let (bank_forks, bank_forks_info, _) =
-            process_blocktree_from_root(&blocktree, bank1, &opts).unwrap();
+            process_blocktree_from_root(&genesis_block, &blocktree, bank1, &opts).unwrap();
 
         assert_eq!(bank_forks_info.len(), 1); // One fork
         assert_eq!(

local_cluster/Cargo.toml

@@ -13,11 +13,13 @@ log = "0.4.8"
 rand = "0.6.5"
 solana-bench-exchange = { path = "../bench-exchange", version = "0.20.0" }
 solana-bench-tps = { path = "../bench-tps", version = "0.20.0" }
+solana-config-api = { path = "../programs/config_api", version = "0.20.0" }
 solana-core = { path = "../core", version = "0.20.0" }
 solana-client = { path = "../client", version = "0.20.0" }
 solana-drone = { path = "../drone", version = "0.20.0" }
 solana-exchange-api = { path = "../programs/exchange_api", version = "0.20.0" }
 solana-exchange-program = { path = "../programs/exchange_program", version = "0.20.0" }
+solana-genesis-programs = { path = "../genesis_programs", version = "0.20.0" }
 solana-ledger = { path = "../ledger", version = "0.20.0" }
 solana-logger = { path = "../logger", version = "0.20.0" }
 solana-move-loader-api = { path = "../programs/move_loader_api", version = "0.20.0", optional = true }
@@ -27,6 +29,7 @@ solana-sdk = { path = "../sdk", version = "0.20.0" }
 solana-stake-api = { path = "../programs/stake_api", version = "0.20.0" }
 solana-storage-api = { path = "../programs/storage_api", version = "0.20.0" }
 solana-storage-program = { path = "../programs/storage_program", version = "0.20.0" }
+solana-vest-api = { path = "../programs/vest_api", version = "0.20.0" }
 solana-vote-api = { path = "../programs/vote_api", version = "0.20.0" }
 symlink = "0.1.0"
 tempfile = "3.1.0"

local_cluster/src/local_cluster.rs

@@ -14,7 +14,7 @@ use solana_sdk::{
     client::SyncClient,
     clock::{DEFAULT_SLOTS_PER_EPOCH, DEFAULT_SLOTS_PER_SEGMENT, DEFAULT_TICKS_PER_SLOT},
     epoch_schedule::EpochSchedule,
-    genesis_block::GenesisBlock,
+    genesis_block::{GenesisBlock, OperatingMode},
     message::Message,
     poh_config::PohConfig,
     pubkey::Pubkey,
@@ -74,6 +74,7 @@ pub struct ClusterConfig {
     pub slots_per_segment: u64,
     pub stakers_slot_offset: u64,
     pub native_instruction_processors: Vec<(String, Pubkey)>,
+    pub operating_mode: OperatingMode,
     pub poh_config: PohConfig,
 }
 
@@ -90,6 +91,7 @@ impl Default for ClusterConfig {
             slots_per_segment: DEFAULT_SLOTS_PER_SEGMENT,
             stakers_slot_offset: DEFAULT_SLOTS_PER_EPOCH,
             native_instruction_processors: vec![],
+            operating_mode: OperatingMode::Development,
             poh_config: PohConfig::default(),
         }
     }
@@ -142,7 +144,19 @@ impl LocalCluster {
         genesis_block.slots_per_segment = config.slots_per_segment;
         genesis_block.epoch_schedule =
             EpochSchedule::custom(config.slots_per_epoch, config.stakers_slot_offset, true);
+        genesis_block.operating_mode = config.operating_mode;
         genesis_block.poh_config = config.poh_config.clone();
+
+        match genesis_block.operating_mode {
+            OperatingMode::SoftLaunch => {
+                genesis_block.native_instruction_processors =
+                    solana_genesis_programs::get(genesis_block.operating_mode, 0).unwrap()
+            }
+            // create_genesis_block_with_leader() assumes OperatingMode::Development so do
+            // nothing...
+            OperatingMode::Development => (),
+        }
+
         genesis_block
             .native_instruction_processors
             .extend_from_slice(&config.native_instruction_processors);

local_cluster/src/tests/local_cluster.rs

@@ -5,6 +5,7 @@ use crate::{
 };
 use log::*;
 use serial_test_derive::serial;
+use solana_client::thin_client::create_client;
 use solana_core::{
     broadcast_stage::BroadcastStageType, gossip_service::discover_cluster,
     validator::ValidatorConfig,
@@ -15,6 +16,7 @@ use solana_sdk::{
     client::SyncClient,
     clock,
     epoch_schedule::{EpochSchedule, MINIMUM_SLOTS_PER_EPOCH},
+    genesis_block::OperatingMode,
     poh_config::PohConfig,
 };
 use std::path::{Path, PathBuf};
@@ -296,6 +298,43 @@ fn test_listener_startup() {
     assert_eq!(cluster_nodes.len(), 4);
 }
 
+#[test]
+#[serial]
+fn test_softlaunch_operating_mode() {
+    solana_logger::setup();
+
+    let config = ClusterConfig {
+        operating_mode: OperatingMode::SoftLaunch,
+        node_stakes: vec![100; 1],
+        cluster_lamports: 1_000,
+        validator_configs: vec![ValidatorConfig::default(); 1],
+        ..ClusterConfig::default()
+    };
+    let cluster = LocalCluster::new(&config);
+    let (cluster_nodes, _) = discover_cluster(&cluster.entry_point_info.gossip, 1).unwrap();
+    assert_eq!(cluster_nodes.len(), 1);
+
+    let client = create_client(
+        cluster.entry_point_info.client_facing_addr(),
+        solana_core::cluster_info::VALIDATOR_PORT_RANGE,
+    );
+
+    // Programs that are not available at soft launch
+    for program_id in [
+        &solana_config_api::id(),
+        &solana_sdk::bpf_loader::id(),
+        &solana_sdk::system_program::id(),
+        &solana_vest_api::id(),
+    ]
+    .iter()
+    {
+        assert_eq!(
+            (program_id, client.get_account(program_id).unwrap()),
+            (program_id, None)
+        );
+    }
+}
+
 #[allow(unused_attributes)]
 #[test]
 #[serial]

multinode-demo/setup.sh

@@ -26,6 +26,7 @@ default_arg --bootstrap-storage-keypair "$SOLANA_CONFIG_DIR"/bootstrap-leader/st
 default_arg --ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader
 default_arg --mint "$SOLANA_CONFIG_DIR"/mint-keypair.json
 default_arg --hashes-per-tick auto
+default_arg --dev
 $solana_genesis "${args[@]}"
 
 (

runtime/src/bank.rs

@@ -151,6 +151,8 @@ impl StatusCacheRc {
     }
 }
 
+pub type EnteredEpochCallback = Box<dyn Fn(&mut Bank) -> () + Sync + Send>;
+
 /// Manager for the state of all accounts and programs after processing its entries.
 #[derive(Default, Deserialize, Serialize)]
 pub struct Bank {
@@ -252,6 +254,11 @@ pub struct Bank {
 
     /// The Message processor
     message_processor: MessageProcessor,
+
+    /// Callback to be notified when a bank enters a new Epoch
+    /// (used to adjust cluster features over time)
+    #[serde(skip)]
+    entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
 }
 
 impl Default for BlockhashQueue {
@@ -339,6 +346,7 @@ impl Bank {
             tick_height: AtomicU64::new(parent.tick_height.load(Ordering::Relaxed)),
             signature_count: AtomicU64::new(0),
             message_processor: MessageProcessor::default(),
+            entered_epoch_callback: parent.entered_epoch_callback.clone(),
         };
 
         datapoint_debug!(
@@ -348,6 +356,15 @@ impl Bank {
         );
 
         let leader_schedule_epoch = epoch_schedule.get_leader_schedule_epoch(slot);
+
+        if parent.epoch() < new.epoch() {
+            if let Some(entered_epoch_callback) =
+                parent.entered_epoch_callback.read().unwrap().as_ref()
+            {
+                entered_epoch_callback(&mut new)
+            }
+        }
+
         // update epoch_stakes cache
         //  if my parent didn't populate for this staker's epoch, we've
         //  crossed a boundary
@@ -1290,6 +1307,13 @@ impl Bank {
         self.rc.parent = RwLock::new(Some(parent.clone()));
     }
 
+    pub fn set_entered_epoch_callback(&self, entered_epoch_callback: EnteredEpochCallback) {
+        std::mem::replace(
+            &mut *self.entered_epoch_callback.write().unwrap(),
+            Some(entered_epoch_callback),
+        );
+    }
+
     pub fn get_account(&self, pubkey: &Pubkey) -> Option<Account> {
         self.rc
             .accounts
@@ -2773,6 +2797,41 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_bank_entered_epoch_callback() {
+        let (genesis_block, _) = create_genesis_block(500);
+        let bank0 = Arc::new(Bank::new(&genesis_block));
+        let callback_count = Arc::new(AtomicU64::new(0));
+
+        bank0.set_entered_epoch_callback({
+            let callback_count = callback_count.clone();
+            //Box::new(move |_bank: &mut Bank| {
+            Box::new(move |_| {
+                callback_count.fetch_add(1, Ordering::SeqCst);
+            })
+        });
+
+        let _bank1 =
+            Bank::new_from_parent(&bank0, &Pubkey::default(), bank0.get_slots_in_epoch(0) - 1);
+        // No callback called while within epoch 0
+        assert_eq!(callback_count.load(Ordering::SeqCst), 0);
+
+        let _bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), bank0.get_slots_in_epoch(0));
+        // Callback called as bank1 is in epoch 1
+        assert_eq!(callback_count.load(Ordering::SeqCst), 1);
+
+        callback_count.store(0, Ordering::SeqCst);
+        let _bank1 = Bank::new_from_parent(
+            &bank0,
+            &Pubkey::default(),
+            std::u64::MAX / bank0.ticks_per_slot - 1,
+        );
+        // If the new bank jumps ahead multiple epochs the callback is still only called once.
+        // This was done to keep the callback implementation simpler as new bank will never jump
+        // cross multiple epochs in a real deployment.
+        assert_eq!(callback_count.load(Ordering::SeqCst), 1);
+    }
+
     #[test]
     fn test_is_delta_true() {
         let (genesis_block, mint_keypair) = create_genesis_block(500);

sdk/src/genesis_block.rs

@@ -21,6 +21,12 @@ use std::{
     path::Path,
 };
 
+#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
+pub enum OperatingMode {
+    SoftLaunch,  // Cluster features incrementally enabled over time
+    Development, // All features (including experimental features) available immediately from genesis
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct GenesisBlock {
     pub accounts: Vec<(Pubkey, Account)>,
@@ -33,6 +39,7 @@ pub struct GenesisBlock {
     pub rent_calculator: RentCalculator,
     pub inflation: Inflation,
     pub epoch_schedule: EpochSchedule,
+    pub operating_mode: OperatingMode,
 }
 
 // useful for basic tests
@@ -63,6 +70,7 @@ impl Default for GenesisBlock {
             fee_calculator: FeeCalculator::default(),
             rent_calculator: RentCalculator::default(),
             epoch_schedule: EpochSchedule::default(),
+            operating_mode: OperatingMode::Development,
         }
     }
 }