Oracle staleness: Check when accessing oracle price

client/src/health_cache.rs

@@ -28,6 +28,7 @@ pub fn new(
         n_perps: active_perp_len,
         begin_perp: active_token_len * 2,
         begin_serum3: active_token_len * 2 + active_perp_len * 2,
+        staleness_slot: None,
     };
     mango_v4::state::new_health_cache(&account.borrow(), &retriever).context("make health cache")
 }

client/src/perp_pnl.rs

@@ -25,8 +25,10 @@ pub fn fetch_top(
     let perp_market =
         account_fetcher_fetch_anchor_account::<PerpMarket>(account_fetcher, &perp.address)?;
     let oracle_acc = account_fetcher.fetch_raw_account(&perp_market.oracle)?;
-    let oracle_price =
+    let oracle_price = perp_market.oracle_price(
-        perp_market.oracle_price(&KeyedAccountSharedData::new(perp_market.oracle, oracle_acc))?;
+        &KeyedAccountSharedData::new(perp_market.oracle, oracle_acc),
+        None,
+    )?;
 
     let accounts =
         account_fetcher.fetch_program_accounts(&mango_v4::id(), MangoAccount::discriminator())?;

liquidator/src/liquidate.rs

@@ -161,10 +161,10 @@ impl<'a> LiquidateHelper<'a> {
                 let oracle = self
                     .account_fetcher
                     .fetch_raw_account(&perp.market.oracle)?;
-                let price = perp.market.oracle_price(&KeyedAccountSharedData::new(
+                let price = perp.market.oracle_price(
-                    perp.market.oracle,
+                    &KeyedAccountSharedData::new(perp.market.oracle, oracle.into()),
-                    oracle.into(),
+                    None,
-                ))?;
+                )?;
                 Ok(Some((
                     pp.market_index,
                     base_lots,
@@ -342,10 +342,10 @@ impl<'a> LiquidateHelper<'a> {
                 let oracle = self
                     .account_fetcher
                     .fetch_raw_account(&token.mint_info.oracle)?;
-                let price = bank.oracle_price(&KeyedAccountSharedData::new(
+                let price = bank.oracle_price(
-                    token.mint_info.oracle,
+                    &KeyedAccountSharedData::new(token.mint_info.oracle, oracle.into()),
-                    oracle.into(),
+                    None,
-                ))?;
+                )?;
                 Ok((
                     token_position.token_index,
                     price,

liquidator/src/rebalance.rs

@@ -45,10 +45,10 @@ impl TokenState {
         account_fetcher: &chain_data::AccountFetcher,
     ) -> anyhow::Result<I80F48> {
         let oracle = account_fetcher.fetch_raw_account(&token.mint_info.oracle)?;
-        bank.oracle_price(&KeyedAccountSharedData::new(
+        bank.oracle_price(
-            token.mint_info.oracle,
+            &KeyedAccountSharedData::new(token.mint_info.oracle, oracle.into()),
-            oracle.into(),
+            None,
-        ))
+        )
         .map_err_anyhow()
     }
 }

programs/mango-v4/src/error.rs

@@ -51,6 +51,10 @@ pub enum MangoError {
     MaxSettleAmountMustBeGreaterThanZero,
     #[msg("the perp position has open orders or unprocessed fill events")]
     HasOpenPerpOrders,
+    #[msg("an oracle does not reach the confidence threshold")]
+    OracleConfidence,
+    #[msg("an oracle is stale")]
+    OracleStale,
 }
 
 pub trait Contextable {

programs/mango-v4/src/instructions/perp_liq_base_position.rs

@@ -82,8 +82,10 @@ pub fn perp_liq_base_position(
     let base_lot_size = I80F48::from(perp_market.base_lot_size);
 
     // Get oracle price for market. Price is validated inside
-    let oracle_price =
+    let oracle_price = perp_market.oracle_price(
-        perp_market.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+        &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+        None, // checked in health
+    )?;
     let price_per_lot = cm!(base_lot_size * oracle_price);
 
     // Fetch perp positions for accounts, creating for the liqor if needed

programs/mango-v4/src/instructions/perp_place_order.rs

@@ -50,8 +50,10 @@ pub fn perp_place_order(ctx: Context<PerpPlaceOrder>, order: Order, limit: u8) -
         let mut perp_market = ctx.accounts.perp_market.load_mut()?;
         let book = ctx.accounts.orderbook.load_mut()?;
 
-        oracle_price =
+        oracle_price = perp_market.oracle_price(
-            perp_market.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+            &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+            None, // staleness checked in health
+        )?;
 
         perp_market.update_funding(&book, oracle_price, now_ts)?;
     }

programs/mango-v4/src/instructions/perp_settle_fees.rs

@@ -54,8 +54,10 @@ pub fn perp_settle_fees(ctx: Context<PerpSettleFees>, max_settle_amount: u64) ->
     );
 
     // Get oracle price for market. Price is validated inside
-    let oracle_price =
+    let oracle_price = perp_market.oracle_price(
-        perp_market.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+        &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+        None, // staleness checked in health
+    )?;
 
     // Fetch perp positions for accounts
     let perp_position = account.perp_position_mut(perp_market.perp_market_index)?;

programs/mango-v4/src/instructions/perp_settle_pnl.rs

@@ -101,8 +101,10 @@ pub fn perp_settle_pnl(ctx: Context<PerpSettlePnl>) -> Result<()> {
     );
 
     // Get oracle price for market. Price is validated inside
-    let oracle_price =
+    let oracle_price = perp_market.oracle_price(
-        perp_market.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+        &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+        None, // staleness checked in health
+    )?;
 
     // Fetch perp positions for accounts
     let a_perp_position = account_a.perp_position_mut(perp_market_index)?;

programs/mango-v4/src/instructions/perp_update_funding.rs

@@ -26,8 +26,11 @@ pub fn perp_update_funding(ctx: Context<PerpUpdateFunding>) -> Result<()> {
     let mut perp_market = ctx.accounts.perp_market.load_mut()?;
     let book = ctx.accounts.orderbook.load_mut()?;
 
-    let oracle_price =
+    let now_slot = Clock::get()?.slot;
-        perp_market.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+    let oracle_price = perp_market.oracle_price(
+        &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+        Some(now_slot),
+    )?;
 
     perp_market.update_funding(&book, oracle_price, now_ts)?;
 

programs/mango-v4/src/instructions/token_deposit.rs

@@ -121,7 +121,10 @@ impl<'a, 'info> DepositCommon<'a, 'info> {
 
         let indexed_position = position.indexed_position;
         let bank = self.bank.load()?;
-        let oracle_price = bank.oracle_price(&AccountInfoRef::borrow(self.oracle.as_ref())?)?;
+        let oracle_price = bank.oracle_price(
+            &AccountInfoRef::borrow(self.oracle.as_ref())?,
+            None, // staleness checked in health
+        )?;
 
         // Update the net deposits - adjust by price so different tokens are on the same basis (in USD terms)
         let amount_usd = cm!(amount_i80f48 * oracle_price).to_num::<i64>();

programs/mango-v4/src/instructions/token_update_index_and_rate.rs

@@ -78,7 +78,8 @@ pub fn token_update_index_and_rate(ctx: Context<TokenUpdateIndexAndRate>) -> Res
         .load()?
         .verify_banks_ais(ctx.remaining_accounts)?;
 
-    let now_ts = Clock::get()?.unix_timestamp;
+    let clock = Clock::get()?;
+    let now_ts = clock.unix_timestamp;
 
     // compute indexed_total
     let mut indexed_total_deposits = I80F48::ZERO;
@@ -106,8 +107,10 @@ pub fn token_update_index_and_rate(ctx: Context<TokenUpdateIndexAndRate>) -> Res
             now_ts,
         );
 
-        let price =
+        let price = some_bank.oracle_price(
-            some_bank.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+            &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+            Some(clock.slot),
+        )?;
         emit!(UpdateIndexLog {
             mango_group: mint_info.group.key(),
             token_index: mint_info.token_index,

programs/mango-v4/src/instructions/token_withdraw.rs

@@ -122,7 +122,11 @@ pub fn token_withdraw(ctx: Context<TokenWithdraw>, amount: u64, allow_borrow: bo
     )?;
 
     let native_position_after = position.native(&bank);
-    let oracle_price = bank.oracle_price(&AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?)?;
+    let now_slot = Clock::get()?.slot;
+    let oracle_price = bank.oracle_price(
+        &AccountInfoRef::borrow(ctx.accounts.oracle.as_ref())?,
+        Some(now_slot),
+    )?;
 
     emit!(TokenBalanceLog {
         mango_group: ctx.accounts.group.key(),

programs/mango-v4/src/state/bank.rs

@@ -583,9 +583,18 @@ impl Bank {
         }
     }
 
-    pub fn oracle_price(&self, oracle_acc: &impl KeyedAccountReader) -> Result<I80F48> {
+    pub fn oracle_price(
+        &self,
+        oracle_acc: &impl KeyedAccountReader,
+        staleness_slot: Option<u64>,
+    ) -> Result<I80F48> {
         require_keys_eq!(self.oracle, *oracle_acc.key());
-        oracle::oracle_price(oracle_acc, &self.oracle_config, self.mint_decimals)
+        oracle::oracle_price(
+            oracle_acc,
+            &self.oracle_config,
+            self.mint_decimals,
+            staleness_slot,
+        )
     }
 }
 

programs/mango-v4/src/state/health.rs

@@ -65,6 +65,7 @@ pub struct FixedOrderAccountRetriever<T: KeyedAccountReader> {
     pub n_perps: usize,
     pub begin_perp: usize,
     pub begin_serum3: usize,
+    pub staleness_slot: Option<u64>,
 }
 
 pub fn new_fixed_order_account_retriever<'a, 'info>(
@@ -85,6 +86,7 @@ pub fn new_fixed_order_account_retriever<'a, 'info>(
         n_perps: active_perp_len,
         begin_perp: cm!(active_token_len * 2),
         begin_serum3: cm!(active_token_len * 2 + active_perp_len * 2),
+        staleness_slot: Some(Clock::get()?.slot),
     })
 }
 
@@ -111,12 +113,12 @@ impl<T: KeyedAccountReader> FixedOrderAccountRetriever<T> {
 
     fn oracle_price(&self, account_index: usize, bank: &Bank) -> Result<I80F48> {
         let oracle = &self.ais[cm!(self.n_banks + account_index)];
-        bank.oracle_price(oracle)
+        bank.oracle_price(oracle, self.staleness_slot)
     }
 
     fn oracle_price_perp(&self, account_index: usize, perp_market: &PerpMarket) -> Result<I80F48> {
         let oracle = &self.ais[self.begin_perp + self.n_perps + account_index];
-        perp_market.oracle_price(oracle)
+        perp_market.oracle_price(oracle, self.staleness_slot)
     }
 }
 
@@ -210,6 +212,7 @@ pub struct ScanningAccountRetriever<'a, 'info> {
     serum3_oos: Vec<AccountInfoRef<'a, 'info>>,
     token_index_map: HashMap<TokenIndex, usize>,
     perp_index_map: HashMap<PerpMarketIndex, usize>,
+    staleness_slot: Option<u64>,
 }
 
 // Returns None if `ai` doesn't have the owner or discriminator for T
@@ -232,6 +235,14 @@ fn can_load_as<'a, T: ZeroCopy + Owner>(
 
 impl<'a, 'info> ScanningAccountRetriever<'a, 'info> {
     pub fn new(ais: &'a [AccountInfo<'info>], group: &Pubkey) -> Result<Self> {
+        Self::new_with_staleness(ais, group, Some(Clock::get()?.slot))
+    }
+
+    pub fn new_with_staleness(
+        ais: &'a [AccountInfo<'info>],
+        group: &Pubkey,
+        staleness_slot: Option<u64>,
+    ) -> Result<Self> {
         // find all Bank accounts
         let mut token_index_map = HashMap::with_capacity(ais.len() / 2);
         ais.iter()
@@ -283,6 +294,7 @@ impl<'a, 'info> ScanningAccountRetriever<'a, 'info> {
             serum3_oos: AccountInfoRef::borrow_slice(&ais[serum3_start..])?,
             token_index_map,
             perp_index_map,
+            staleness_slot,
         })
     }
 
@@ -312,7 +324,7 @@ impl<'a, 'info> ScanningAccountRetriever<'a, 'info> {
             let index = self.bank_index(token_index1)?;
             let bank = self.banks[index].load_mut_fully_unchecked::<Bank>()?;
             let oracle = &self.oracles[index];
-            let price = bank.oracle_price(oracle)?;
+            let price = bank.oracle_price(oracle, self.staleness_slot)?;
             return Ok((bank, price, None));
         }
         let index1 = self.bank_index(token_index1)?;
@@ -330,8 +342,8 @@ impl<'a, 'info> ScanningAccountRetriever<'a, 'info> {
         let bank2 = second_bank_part[second - (first + 1)].load_mut_fully_unchecked::<Bank>()?;
         let oracle1 = &self.oracles[first];
         let oracle2 = &self.oracles[second];
-        let price1 = bank1.oracle_price(oracle1)?;
+        let price1 = bank1.oracle_price(oracle1, self.staleness_slot)?;
-        let price2 = bank2.oracle_price(oracle2)?;
+        let price2 = bank2.oracle_price(oracle2, self.staleness_slot)?;
         if swap {
             Ok((bank2, price2, Some((bank1, price1))))
         } else {
@@ -343,7 +355,7 @@ impl<'a, 'info> ScanningAccountRetriever<'a, 'info> {
         let index = self.bank_index(token_index)?;
         let bank = self.banks[index].load_fully_unchecked::<Bank>()?;
         let oracle = &self.oracles[index];
-        Ok((bank, bank.oracle_price(oracle)?))
+        Ok((bank, bank.oracle_price(oracle, self.staleness_slot)?))
     }
 
     pub fn scanned_perp_market_and_oracle(
@@ -353,7 +365,7 @@ impl<'a, 'info> ScanningAccountRetriever<'a, 'info> {
         let index = self.perp_market_index(perp_market_index)?;
         let perp_market = self.perp_markets[index].load_fully_unchecked::<PerpMarket>()?;
         let oracle_acc = &self.perp_oracles[index];
-        let oracle_price = perp_market.oracle_price(oracle_acc)?;
+        let oracle_price = perp_market.oracle_price(oracle_acc, self.staleness_slot)?;
         Ok((perp_market, oracle_price))
     }
 
@@ -1481,7 +1493,7 @@ mod tests {
             oo1.as_account_info(),
         ];
 
-        let retriever = ScanningAccountRetriever::new(&ais, &group).unwrap();
+        let retriever = ScanningAccountRetriever::new_with_staleness(&ais, &group, None).unwrap();
 
         // for bank1/oracle1, including open orders (scenario: bids execute)
         let health1 = (100.0 + 1.0 + 2.0 + (20.0 + 15.0 * 5.0)) * 0.8;
@@ -1532,7 +1544,8 @@ mod tests {
             oo1.as_account_info(),
         ];
 
-        let mut retriever = ScanningAccountRetriever::new(&ais, &group).unwrap();
+        let mut retriever =
+            ScanningAccountRetriever::new_with_staleness(&ais, &group, None).unwrap();
 
         assert_eq!(retriever.banks.len(), 3);
         assert_eq!(retriever.token_index_map.len(), 3);
@@ -1678,7 +1691,7 @@ mod tests {
             oo2.as_account_info(),
         ];
 
-        let retriever = ScanningAccountRetriever::new(&ais, &group).unwrap();
+        let retriever = ScanningAccountRetriever::new_with_staleness(&ais, &group, None).unwrap();
 
         assert!(health_eq(
             compute_health(&account.borrow(), HealthType::Init, &retriever).unwrap(),
@@ -2166,7 +2179,7 @@ mod tests {
             oracle1_ai,
         ];
 
-        let retriever = ScanningAccountRetriever::new(&ais, &group).unwrap();
+        let retriever = ScanningAccountRetriever::new_with_staleness(&ais, &group, None).unwrap();
 
         assert!(health_eq(
             compute_health(&account.borrow(), HealthType::Init, &retriever).unwrap(),
@@ -2207,7 +2220,7 @@ mod tests {
             oo1.as_account_info(),
         ];
 
-        let retriever = ScanningAccountRetriever::new(&ais, &group).unwrap();
+        let retriever = ScanningAccountRetriever::new_with_staleness(&ais, &group, None).unwrap();
         let result = retriever.perp_market_and_oracle_price(&group, 0, 9);
         assert!(result.is_err());
     }

programs/mango-v4/src/state/oracle.rs

@@ -131,33 +131,59 @@ pub fn determine_oracle_type(acc_info: &impl KeyedAccountReader) -> Result<Oracl
 /// Example: The for SOL at 40 USDC/SOL it would return 0.04 (the unit is USDC-native/SOL-native)
 ///
 /// This currently assumes that quote decimals is 6, like for USDC.
+///
+/// Pass `staleness_slot` = None to skip the staleness check
 pub fn oracle_price(
     acc_info: &impl KeyedAccountReader,
     config: &OracleConfig,
     base_decimals: u8,
+    staleness_slot: Option<u64>,
 ) -> Result<I80F48> {
     let data = &acc_info.data();
     let oracle_type = determine_oracle_type(acc_info)?;
+    let staleness_slot = staleness_slot.unwrap_or(0);
 
     Ok(match oracle_type {
         OracleType::Stub => acc_info.load::<StubOracle>()?.price,
         OracleType::Pyth => {
-            let price_account = pyth_sdk_solana::load_price(data).unwrap();
+            let price_account = pyth_sdk_solana::state::load_price_account(data).unwrap();
-            let price = I80F48::from_num(price_account.price);
+            let price_data = price_account.to_price();
+            let price = I80F48::from_num(price_data.price);
 
             // Filter out bad prices
-            if I80F48::from_num(price_account.conf) > cm!(config.conf_filter * price) {
+            if I80F48::from_num(price_data.conf) > cm!(config.conf_filter * price) {
                 msg!(
-                    "Pyth conf interval too high; pubkey {} price: {} price_account.conf: {}",
+                    "Pyth conf interval too high; pubkey {} price: {} price_data.conf: {}",
                     acc_info.key(),
                     price.to_num::<f64>(),
-                    price_account.conf
+                    price_data.conf
                 );
 
                 // future: in v3, we had pricecache, and in case of luna, when there were no updates, we used last known value from cache
                 // we'll have to add a CachedOracle that is based on one of the oracle types, needs a separate keeper and supports
                 // maintaining this "last known good value"
-                return Err(MangoError::SomeError.into());
+                return Err(MangoError::OracleConfidence.into());
+            }
+
+            // The aggregation pub slot is the time that the aggregate was computed from published data
+            // that was at most 25 slots old. That means it underestimates the actual staleness, potentially
+            // significantly.
+            let agg_pub_slot = price_account.agg.pub_slot;
+            if config.max_staleness_slots >= 0
+                && price_account
+                    .agg
+                    .pub_slot
+                    .saturating_add(config.max_staleness_slots as u64)
+                    < staleness_slot
+            {
+                msg!(
+                    "Pyth price too stale; pubkey {} price: {} pub slot: {}",
+                    acc_info.key(),
+                    price.to_num::<f64>(),
+                    agg_pub_slot,
+                );
+
+                return Err(MangoError::OracleStale.into());
             }
 
             let decimals = cm!((price_account.expo as i8) + QUOTE_DECIMALS - (base_decimals as i8));
@@ -187,7 +213,23 @@ pub fn oracle_price(
                     price.to_num::<f64>(),
                     std_deviation_decimal
                 );
-                return Err(MangoError::SomeError.into());
+                return Err(MangoError::OracleConfidence.into());
+            }
+
+            // The round_open_slot is an overestimate of the oracle staleness: Reporters will see
+            // the round opening and only then start executing the price tasks.
+            let round_open_slot = feed.latest_confirmed_round.round_open_slot;
+            if config.max_staleness_slots >= 0
+                && round_open_slot.saturating_add(config.max_staleness_slots as u64)
+                    < staleness_slot
+            {
+                msg!(
+                    "Switchboard v2 price too stale; pubkey {} price: {} latest_confirmed_round.round_open_slot: {}",
+                    acc_info.key(),
+                    price.to_num::<f64>(),
+                    round_open_slot,
+                );
+                return Err(MangoError::OracleConfidence.into());
             }
 
             let decimals = cm!(QUOTE_DECIMALS - (base_decimals as i8));
@@ -209,7 +251,21 @@ pub fn oracle_price(
                     min_response,
                     max_response
                 );
-                return Err(MangoError::SomeError.into());
+                return Err(MangoError::OracleConfidence.into());
+            }
+
+            let round_open_slot = result.result.round_open_slot;
+            if config.max_staleness_slots >= 0
+                && round_open_slot.saturating_add(config.max_staleness_slots as u64)
+                    < staleness_slot
+            {
+                msg!(
+                    "Switchboard v1 price too stale; pubkey {} price: {} round_open_slot: {}",
+                    acc_info.key(),
+                    price.to_num::<f64>(),
+                    round_open_slot,
+                );
+                return Err(MangoError::OracleConfidence.into());
             }
 
             let decimals = cm!(QUOTE_DECIMALS - (base_decimals as i8));

programs/mango-v4/src/state/perp_market.rs

@@ -139,9 +139,18 @@ impl PerpMarket {
         orderbook::new_node_key(side, price_data, self.seq_num)
     }
 
-    pub fn oracle_price(&self, oracle_acc: &impl KeyedAccountReader) -> Result<I80F48> {
+    pub fn oracle_price(
+        &self,
+        oracle_acc: &impl KeyedAccountReader,
+        staleness_slot: Option<u64>,
+    ) -> Result<I80F48> {
         require_keys_eq!(self.oracle, *oracle_acc.key());
-        oracle::oracle_price(oracle_acc, &self.oracle_config, self.base_decimals)
+        oracle::oracle_price(
+            oracle_acc,
+            &self.oracle_config,
+            self.base_decimals,
+            staleness_slot,
+        )
     }
 
     /// Use current order book price and index price to update the instantaneous funding