Settler: Keep going on error (#873)
in particular: don't panic when a health cache can't be constructed
This commit is contained in:
parent
9f4cb0f776
commit
712a2e3bd6
|
@ -308,7 +308,9 @@ async fn main() -> anyhow::Result<()> {
|
||||||
account_addresses = state.mango_accounts.iter().cloned().collect();
|
account_addresses = state.mango_accounts.iter().cloned().collect();
|
||||||
}
|
}
|
||||||
|
|
||||||
settlement.settle(account_addresses).await.unwrap();
|
if let Err(err) = settlement.settle(account_addresses).await {
|
||||||
|
warn!("settle error: {err:?}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -329,7 +331,9 @@ async fn main() -> anyhow::Result<()> {
|
||||||
account_addresses = state.mango_accounts.iter().cloned().collect();
|
account_addresses = state.mango_accounts.iter().cloned().collect();
|
||||||
}
|
}
|
||||||
|
|
||||||
tcs_start.run_pass(account_addresses).await.unwrap();
|
if let Err(err) = tcs_start.run_pass(account_addresses).await {
|
||||||
|
warn!("tcs-start error: {err:?}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
@ -96,7 +96,13 @@ impl SettlementState {
|
||||||
let mut all_negative_settleable =
|
let mut all_negative_settleable =
|
||||||
HashMap::<PerpMarketIndex, priority_queue::PriorityQueue<Pubkey, I80F48>>::new();
|
HashMap::<PerpMarketIndex, priority_queue::PriorityQueue<Pubkey, I80F48>>::new();
|
||||||
for account_key in accounts.iter() {
|
for account_key in accounts.iter() {
|
||||||
let mut account = account_fetcher.fetch_mango_account(account_key)?;
|
let mut account = match account_fetcher.fetch_mango_account(account_key) {
|
||||||
|
Ok(acc) => acc,
|
||||||
|
Err(e) => {
|
||||||
|
info!("could not fetch account, skipping {account_key}: {e:?}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
if account.fixed.group != mango_client.group() {
|
if account.fixed.group != mango_client.group() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -111,10 +117,10 @@ impl SettlementState {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let health_cache = mango_client
|
let health_cache = match mango_client.health_cache(&account).await {
|
||||||
.health_cache(&account)
|
Ok(hc) => hc,
|
||||||
.await
|
Err(_) => continue, // Skip for stale/unconfident oracles
|
||||||
.context("creating health cache")?;
|
};
|
||||||
let liq_end_health = health_cache.health(HealthType::LiquidationEnd);
|
let liq_end_health = health_cache.health(HealthType::LiquidationEnd);
|
||||||
|
|
||||||
for perp_market_index in perp_indexes {
|
for perp_market_index in perp_indexes {
|
||||||
|
@ -123,14 +129,19 @@ impl SettlementState {
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
None => continue, // skip accounts with perp positions where we couldn't get the price and market
|
None => continue, // skip accounts with perp positions where we couldn't get the price and market
|
||||||
};
|
};
|
||||||
let perp_max_settle =
|
let perp_max_settle = health_cache
|
||||||
health_cache.perp_max_settle(perp_market.settle_token_index)?;
|
.perp_max_settle(perp_market.settle_token_index)
|
||||||
|
.expect("perp_max_settle always succeeds when the token index is valid");
|
||||||
|
|
||||||
let perp_position = account.perp_position_mut(perp_market_index).unwrap();
|
let perp_position = account
|
||||||
|
.perp_position_mut(perp_market_index)
|
||||||
|
.expect("index comes from active_perp_positions()");
|
||||||
perp_position.settle_funding(perp_market);
|
perp_position.settle_funding(perp_market);
|
||||||
perp_position.update_settle_limit(perp_market, now_ts);
|
perp_position.update_settle_limit(perp_market, now_ts);
|
||||||
|
|
||||||
let unsettled = perp_position.unsettled_pnl(perp_market, *perp_price)?;
|
let unsettled = perp_position
|
||||||
|
.unsettled_pnl(perp_market, *perp_price)
|
||||||
|
.expect("unsettled_pnl always succeeds with the right perp market");
|
||||||
let limited = perp_position.apply_pnl_settle_limit(perp_market, unsettled);
|
let limited = perp_position.apply_pnl_settle_limit(perp_market, unsettled);
|
||||||
let settleable = if limited >= 0 {
|
let settleable = if limited >= 0 {
|
||||||
limited
|
limited
|
||||||
|
@ -157,7 +168,7 @@ impl SettlementState {
|
||||||
liq_end_health,
|
liq_end_health,
|
||||||
maint_health,
|
maint_health,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.expect("always ok");
|
||||||
|
|
||||||
// Assume that settle_fee_flat is near the tx fee, and if we can't possibly
|
// Assume that settle_fee_flat is near the tx fee, and if we can't possibly
|
||||||
// make up for the tx fee even with multiple settle ix in one tx, skip.
|
// make up for the tx fee even with multiple settle ix in one tx, skip.
|
||||||
|
@ -181,7 +192,9 @@ impl SettlementState {
|
||||||
let address_lookup_tables = mango_client.mango_address_lookup_tables().await?;
|
let address_lookup_tables = mango_client.mango_address_lookup_tables().await?;
|
||||||
|
|
||||||
for (perp_market_index, mut positive_settleable) in all_positive_settleable {
|
for (perp_market_index, mut positive_settleable) in all_positive_settleable {
|
||||||
let (perp_market, _, _) = perp_market_info.get(&perp_market_index).unwrap();
|
let (perp_market, _, _) = perp_market_info
|
||||||
|
.get(&perp_market_index)
|
||||||
|
.expect("perp market must exist");
|
||||||
let negative_settleable = match all_negative_settleable.get_mut(&perp_market_index) {
|
let negative_settleable = match all_negative_settleable.get_mut(&perp_market_index) {
|
||||||
None => continue,
|
None => continue,
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
|
@ -286,15 +299,17 @@ impl<'a> SettleBatchProcessor<'a> {
|
||||||
|
|
||||||
let send_result = self.mango_client.client.send_transaction(&tx).await;
|
let send_result = self.mango_client.client.send_transaction(&tx).await;
|
||||||
|
|
||||||
if let Err(err) = send_result {
|
match send_result {
|
||||||
info!("error while sending settle batch: {}", err);
|
Ok(txsig) => {
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
let txsig = send_result.unwrap();
|
|
||||||
info!("sent settle tx: {txsig}");
|
info!("sent settle tx: {txsig}");
|
||||||
Ok(Some(txsig))
|
Ok(Some(txsig))
|
||||||
}
|
}
|
||||||
|
Err(err) => {
|
||||||
|
info!("error while sending settle batch: {}", err);
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn add_and_maybe_send(
|
async fn add_and_maybe_send(
|
||||||
&mut self,
|
&mut self,
|
||||||
|
|
|
@ -57,7 +57,13 @@ impl State {
|
||||||
|
|
||||||
let mut startable = vec![];
|
let mut startable = vec![];
|
||||||
for account_key in accounts.iter() {
|
for account_key in accounts.iter() {
|
||||||
let account = account_fetcher.fetch_mango_account(account_key).unwrap();
|
let account = match account_fetcher.fetch_mango_account(account_key) {
|
||||||
|
Ok(acc) => acc,
|
||||||
|
Err(e) => {
|
||||||
|
info!("could not fetch account, skipping {account_key}: {e:?}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
if account.fixed.group != mango_client.group() {
|
if account.fixed.group != mango_client.group() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -96,6 +102,11 @@ impl State {
|
||||||
let mut ix_targets = vec![];
|
let mut ix_targets = vec![];
|
||||||
let mut liqor_account = mango_client.mango_account().await?;
|
let mut liqor_account = mango_client.mango_account().await?;
|
||||||
for (pubkey, tcs_id, incentive_token_index) in startable_chunk {
|
for (pubkey, tcs_id, incentive_token_index) in startable_chunk {
|
||||||
|
// can only batch until all token positions are full
|
||||||
|
if let Err(_) = liqor_account.ensure_token_position(*incentive_token_index) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
let ixs = match self.make_start_ix(pubkey, *tcs_id).await {
|
let ixs = match self.make_start_ix(pubkey, *tcs_id).await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
@ -109,7 +120,6 @@ impl State {
|
||||||
};
|
};
|
||||||
instructions.append(ixs);
|
instructions.append(ixs);
|
||||||
ix_targets.push((*pubkey, *tcs_id));
|
ix_targets.push((*pubkey, *tcs_id));
|
||||||
liqor_account.ensure_token_position(*incentive_token_index)?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear newly created token positions, so the liqor account is mostly empty
|
// Clear newly created token positions, so the liqor account is mostly empty
|
||||||
|
@ -120,9 +130,13 @@ impl State {
|
||||||
.collect_vec();
|
.collect_vec();
|
||||||
for token_index in new_token_pos_indices {
|
for token_index in new_token_pos_indices {
|
||||||
let mint = mango_client.context.token(token_index).mint;
|
let mint = mango_client.context.token(token_index).mint;
|
||||||
let ix = mango_client
|
let ix = match mango_client
|
||||||
.token_withdraw_instructions(&liqor_account, mint, u64::MAX, false)
|
.token_withdraw_instructions(&liqor_account, mint, u64::MAX, false)
|
||||||
.await?;
|
.await
|
||||||
|
{
|
||||||
|
Ok(ix) => ix,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
|
||||||
instructions.append(ix)
|
instructions.append(ix)
|
||||||
}
|
}
|
||||||
|
@ -165,7 +179,7 @@ impl State {
|
||||||
pubkey: &Pubkey,
|
pubkey: &Pubkey,
|
||||||
tcs_id: u64,
|
tcs_id: u64,
|
||||||
) -> anyhow::Result<PreparedInstructions> {
|
) -> anyhow::Result<PreparedInstructions> {
|
||||||
let account = self.account_fetcher.fetch_mango_account(pubkey).unwrap();
|
let account = self.account_fetcher.fetch_mango_account(pubkey)?;
|
||||||
self.mango_client
|
self.mango_client
|
||||||
.token_conditional_swap_start_instruction((pubkey, &account), tcs_id)
|
.token_conditional_swap_start_instruction((pubkey, &account), tcs_id)
|
||||||
.await
|
.await
|
||||||
|
|
Loading…
Reference in New Issue