Add intermittent Timestamping to Votes (#7233)

* Add intermittent timestamp to Vote

* Add timestamp to VoteState, add timestamp processing to program

* Print recent timestamp with solana show-vote-account

* Add offset of 1 to timestamp Vote interval to initialize at node boot (slot 1)

* Review comments

* Cache last_timestamp in Tower and use for interval check

* Move work into Tower method

* Clarify timestamping interval

* Replace tuple with struct

cli/src/cli.rs

@@ -637,6 +637,8 @@ fn process_balance(
     use_lamports_unit: bool,
 ) -> ProcessResult {
     let pubkey = pubkey.unwrap_or(config.keypair.pubkey());
+    let string = solana_stake_program::id().to_string();
+    println!("{:}", string);
     let balance = rpc_client.retry_get_balance(&pubkey, 5)?;
     match balance {
         Some(lamports) => Ok(build_balance_message(lamports, use_lamports_unit, true)),

cli/src/vote.rs

@@ -353,6 +353,7 @@ pub fn process_show_vote_account(
             None => "~".to_string(),
         }
     );
+    println!("recent timestamp: {:?}", vote_state.last_timestamp);
     if !vote_state.votes.is_empty() {
         println!("recent votes:");
         for vote in &vote_state.votes {

core/src/consensus.rs

@@ -1,8 +1,16 @@
+use chrono::prelude::*;
 use solana_ledger::bank_forks::BankForks;
 use solana_metrics::datapoint_debug;
 use solana_runtime::bank::Bank;
-use solana_sdk::{account::Account, clock::Slot, hash::Hash, pubkey::Pubkey};
-use solana_vote_program::vote_state::{Lockout, Vote, VoteState, MAX_LOCKOUT_HISTORY};
+use solana_sdk::{
+    account::Account,
+    clock::{Slot, UnixTimestamp},
+    hash::Hash,
+    pubkey::Pubkey,
+};
+use solana_vote_program::vote_state::{
+    BlockTimestamp, Lockout, Vote, VoteState, MAX_LOCKOUT_HISTORY, TIMESTAMP_SLOT_INTERVAL,
+};
 use std::{
     collections::{HashMap, HashSet},
     sync::Arc,
@@ -36,6 +44,7 @@ pub struct Tower {
     threshold_size: f64,
     lockouts: VoteState,
     last_vote: Vote,
+    last_timestamp: BlockTimestamp,
 }
 
 impl Tower {
@@ -46,6 +55,7 @@ impl Tower {
             threshold_size: VOTE_THRESHOLD_SIZE,
             lockouts: VoteState::default(),
             last_vote: Vote::default(),
+            last_timestamp: BlockTimestamp::default(),
         };
 
         tower.initialize_lockouts_from_bank_forks(&bank_forks, vote_account_pubkey);
@@ -180,10 +190,7 @@ impl Tower {
         last_bank_slot: Option<Slot>,
     ) -> (Vote, usize) {
         let mut local_vote_state = local_vote_state.clone();
-        let vote = Vote {
-            slots: vec![slot],
-            hash,
-        };
+        let vote = Vote::new(vec![slot], hash);
         local_vote_state.process_vote_unchecked(&vote);
         let slots = if let Some(last_bank_slot) = last_bank_slot {
             local_vote_state
@@ -201,7 +208,7 @@ impl Tower {
             slots,
             local_vote_state.votes
         );
-        (Vote { slots, hash }, local_vote_state.votes.len() - 1)
+        (Vote::new(slots, hash), local_vote_state.votes.len() - 1)
     }
 
     fn last_bank_vote(bank: &Bank, vote_account_pubkey: &Pubkey) -> Option<Slot> {
@@ -235,10 +242,7 @@ impl Tower {
     }
 
     pub fn record_vote(&mut self, slot: Slot, hash: Hash) -> Option<Slot> {
-        let vote = Vote {
-            slots: vec![slot],
-            hash,
-        };
+        let vote = Vote::new(vec![slot], hash);
         self.record_bank_vote(vote)
     }
 
@@ -246,6 +250,13 @@ impl Tower {
         self.last_vote.clone()
     }
 
+    pub fn last_vote_and_timestamp(&mut self) -> Vote {
+        let mut last_vote = self.last_vote();
+        let current_slot = last_vote.slots.iter().max().unwrap_or(&0);
+        last_vote.timestamp = self.maybe_timestamp(*current_slot);
+        last_vote
+    }
+
     pub fn root(&self) -> Option<Slot> {
         self.lockouts.root_slot
     }
@@ -418,11 +429,27 @@ impl Tower {
             }
         }
     }
+
+    fn maybe_timestamp(&mut self, current_slot: Slot) -> Option<UnixTimestamp> {
+        if self.last_timestamp.slot == 0
+            || self.last_timestamp.slot + TIMESTAMP_SLOT_INTERVAL <= current_slot
+        {
+            let timestamp = Utc::now().timestamp();
+            self.last_timestamp = BlockTimestamp {
+                slot: current_slot,
+                timestamp,
+            };
+            Some(timestamp)
+        } else {
+            None
+        }
+    }
 }
 
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::{thread::sleep, time::Duration};
 
     fn gen_stakes(stake_votes: &[(u64, &[u64])]) -> Vec<(Pubkey, (u64, Account))> {
         let mut stakes = vec![];
@@ -791,6 +818,7 @@ mod test {
         let vote = Vote {
             slots: vec![0],
             hash: Hash::default(),
+            timestamp: None,
         };
         local.process_vote_unchecked(&vote);
         assert_eq!(local.votes.len(), 1);
@@ -805,6 +833,7 @@ mod test {
         let vote = Vote {
             slots: vec![0],
             hash: Hash::default(),
+            timestamp: None,
         };
         local.process_vote_unchecked(&vote);
         assert_eq!(local.votes.len(), 1);
@@ -892,4 +921,21 @@ mod test {
     fn test_recent_votes_exact() {
         vote_and_check_recent(5)
     }
+
+    #[test]
+    fn test_maybe_timestamp() {
+        let mut tower = Tower::default();
+        assert!(tower.maybe_timestamp(TIMESTAMP_SLOT_INTERVAL).is_some());
+        let BlockTimestamp { slot, timestamp } = tower.last_timestamp;
+
+        assert_eq!(tower.maybe_timestamp(1), None);
+        assert_eq!(tower.maybe_timestamp(slot), None);
+        assert_eq!(tower.maybe_timestamp(slot + 1), None);
+
+        sleep(Duration::from_secs(1));
+        assert!(tower
+            .maybe_timestamp(slot + TIMESTAMP_SLOT_INTERVAL + 1)
+            .is_some());
+        assert!(tower.last_timestamp.timestamp > timestamp);
+    }
 }

core/src/replay_stage.rs

@@ -31,14 +31,14 @@ use solana_sdk::{
 };
 use solana_vote_program::vote_instruction;
 use std::{
-    collections::HashMap,
-    collections::HashSet,
-    sync::atomic::{AtomicBool, Ordering},
-    sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender},
-    sync::{Arc, Mutex, RwLock},
+    collections::{HashMap, HashSet},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::{channel, Receiver, RecvTimeoutError, Sender},
+        Arc, Mutex, RwLock,
+    },
     thread::{self, Builder, JoinHandle},
-    time::Duration,
-    time::Instant,
+    time::{Duration, Instant},
 };
 
 pub const MAX_ENTRY_RECV_PER_ITER: usize = 512;
@@ -655,8 +655,11 @@ impl ReplayStage {
             let node_keypair = cluster_info.read().unwrap().keypair.clone();
 
             // Send our last few votes along with the new one
-            let vote_ix =
-                vote_instruction::vote(&vote_account, &voting_keypair.pubkey(), tower.last_vote());
+            let vote_ix = vote_instruction::vote(
+                &vote_account,
+                &voting_keypair.pubkey(),
+                tower.last_vote_and_timestamp(),
+            );
 
             let mut vote_tx =
                 Transaction::new_with_payer(vec![vote_ix], Some(&node_keypair.pubkey()));

programs/vote/src/vote_instruction.rs

@@ -33,6 +33,9 @@ pub enum VoteError {
 
     #[error("vote has no slots, invalid")]
     EmptySlots,
+
+    #[error("vote timestamp not recent")]
+    TimestampTooOld,
 }
 impl<E> DecodeError<E> for VoteError {
     fn type_of() -> &'static str {

programs/vote/src/vote_state.rs

@@ -8,7 +8,7 @@ use serde_derive::{Deserialize, Serialize};
 use solana_sdk::{
     account::{Account, KeyedAccount},
     account_utils::State,
-    clock::{Epoch, Slot},
+    clock::{Epoch, Slot, UnixTimestamp},
     hash::Hash,
     instruction::InstructionError,
     pubkey::Pubkey,
@@ -26,17 +26,27 @@ pub const INITIAL_LOCKOUT: usize = 2;
 //  smaller numbers makes
 pub const MAX_EPOCH_CREDITS_HISTORY: usize = 64;
 
+// Frequency of timestamp Votes In v0.22.0, this is approximately 30min with cluster clock
+// defaults, intended to limit block time drift to < 1hr
+pub const TIMESTAMP_SLOT_INTERVAL: u64 = 4500;
+
 #[derive(Serialize, Default, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct Vote {
     /// A stack of votes starting with the oldest vote
     pub slots: Vec<Slot>,
     /// signature of the bank's state at the last slot
     pub hash: Hash,
+    /// processing timestamp of last slot
+    pub timestamp: Option<UnixTimestamp>,
 }
 
 impl Vote {
     pub fn new(slots: Vec<Slot>, hash: Hash) -> Self {
-        Self { slots, hash }
+        Self {
+            slots,
+            hash,
+            timestamp: None,
+        }
     }
 }
 
@@ -83,6 +93,12 @@ pub enum VoteAuthorize {
     Withdrawer,
 }
 
+#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct BlockTimestamp {
+    pub slot: Slot,
+    pub timestamp: UnixTimestamp,
+}
+
 #[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone)]
 pub struct VoteState {
     /// the node that votes in this account
@@ -109,6 +125,9 @@ pub struct VoteState {
     /// history of how many credits earned by the end of each epoch
     ///  each tuple is (Epoch, credits, prev_credits)
     epoch_credits: Vec<(Epoch, u64, u64)>,
+
+    /// most recent timestamp submitted with a vote
+    pub last_timestamp: BlockTimestamp,
 }
 
 impl VoteState {
@@ -339,6 +358,21 @@ impl VoteState {
             }
         }
     }
+
+    pub fn process_timestamp(
+        &mut self,
+        slot: Slot,
+        timestamp: UnixTimestamp,
+    ) -> Result<(), VoteError> {
+        if (slot < self.last_timestamp.slot || timestamp < self.last_timestamp.timestamp)
+            || ((slot == self.last_timestamp.slot || timestamp == self.last_timestamp.timestamp)
+                && BlockTimestamp { slot, timestamp } != self.last_timestamp)
+        {
+            return Err(VoteError::TimestampTooOld);
+        }
+        self.last_timestamp = BlockTimestamp { slot, timestamp };
+        Ok(())
+    }
 }
 
 /// Authorize the given pubkey to withdraw or sign votes. This may be called multiple times,
@@ -444,6 +478,14 @@ pub fn process_vote(
     verify_authorized_signer(&vote_state.authorized_voter, signers)?;
 
     vote_state.process_vote(vote, slot_hashes, clock.epoch)?;
+    if let Some(timestamp) = vote.timestamp {
+        vote.slots
+            .iter()
+            .max()
+            .ok_or_else(|| VoteError::EmptySlots)
+            .and_then(|slot| vote_state.process_timestamp(*slot, timestamp))
+            .map_err(|err| InstructionError::CustomError(err as u32))?;
+    }
     vote_account.set_state(&vote_state)
 }
 
@@ -1259,4 +1301,48 @@ mod tests {
             1
         );
     }
+
+    #[test]
+    fn test_vote_process_timestamp() {
+        let (slot, timestamp) = (15, 1575412285);
+        let mut vote_state = VoteState::default();
+        vote_state.last_timestamp = BlockTimestamp { slot, timestamp };
+
+        assert_eq!(
+            vote_state.process_timestamp(slot - 1, timestamp + 1),
+            Err(VoteError::TimestampTooOld)
+        );
+        assert_eq!(
+            vote_state.last_timestamp,
+            BlockTimestamp { slot, timestamp }
+        );
+        assert_eq!(
+            vote_state.process_timestamp(slot + 1, timestamp - 1),
+            Err(VoteError::TimestampTooOld)
+        );
+        assert_eq!(
+            vote_state.process_timestamp(slot + 1, timestamp),
+            Err(VoteError::TimestampTooOld)
+        );
+        assert_eq!(
+            vote_state.process_timestamp(slot, timestamp + 1),
+            Err(VoteError::TimestampTooOld)
+        );
+        assert_eq!(vote_state.process_timestamp(slot, timestamp), Ok(()));
+        assert_eq!(
+            vote_state.last_timestamp,
+            BlockTimestamp { slot, timestamp }
+        );
+        assert_eq!(
+            vote_state.process_timestamp(slot + 1, timestamp + 1),
+            Ok(())
+        );
+        assert_eq!(
+            vote_state.last_timestamp,
+            BlockTimestamp {
+                slot: slot + 1,
+                timestamp: timestamp + 1
+            }
+        );
+    }
 }

sdk/src/timing.rs

@@ -38,7 +38,7 @@ pub fn years_as_slots(years: f64, tick_duration: &Duration, ticks_per_slot: u64)
         / ticks_per_slot as f64
 }
 
-/// From slots per year to tick_duration
+/// From slots per year to slot duration
 pub fn slot_duration_from_slots_per_year(slots_per_year: f64) -> Duration {
     // Regarding division by zero potential below: for some reason, if Rust stores an `inf` f64 and
     // then converts it to a u64 on use, it always returns 0, as opposed to std::u64::MAX or any