[accumulator-updater 2/x] Manual Serialization & Zero-copy for Mock-cpi-program (#729)

* feat(accumulator_updater): initial skeleton for accumulator_udpater program

Initial layout for accumulator updater program. Includes mock-cpi-caller which is meant to represent
pyth oracle(or any future allowed program) that will call the accumulator updater program. All
implementation details are open for discussion/subject to change

* test(accumulator_updater): add additional checks in tests and minor clean up

* chore(accumulator_updater): misc clean-up

* refactor(accumulator_updater): added comments & to-dos and minor refactoring to address PR comments

* feat(accumulator_updater): nmanual serialization for mock-cpi-caller schemas & use zero-copy

* chore(accumulator-updater): misc clean-up

* refactor(accumulator-updater): rename parameter in accumulator-updater::initalize ix for consistency

* style(accumulator-updater): switch PriceAccountType enum variants to camelcase

* refactor(accumulator-updater): address PR comments

rename schema to message & associated price messages, remove unncessary
comments, changed addAllowedProgram to setAllowedPrograms

* refactor(accumulator-updater): address more PR comments

consolidate SetAllowedPrograms and UpdateWhitelistAuthority into one context

* style(accumulator-updater): minor style fixes to address PR comments
This commit is contained in:
swimricky 2023-04-04 06:34:14 -07:00 committed by GitHub
parent 2db5a26752
commit cb44e15f33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 328 additions and 129 deletions

View File

@ -899,6 +899,7 @@ version = "0.1.0"
dependencies = [
"accumulator_updater",
"anchor-lang",
"bytemuck",
]
[[package]]

View File

@ -18,34 +18,43 @@ declare_id!("Fg6PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
pub mod accumulator_updater {
use super::*;
pub fn initialize(ctx: Context<Initialize>) -> Result<()> {
pub fn initialize(ctx: Context<Initialize>, authority: Pubkey) -> Result<()> {
require_keys_neq!(authority, Pubkey::default());
let whitelist = &mut ctx.accounts.whitelist;
whitelist.bump = *ctx.bumps.get("whitelist").unwrap();
whitelist.authority = authority;
Ok(())
}
//TODO: add authorization mechanism for this
pub fn add_allowed_program(
ctx: Context<AddAllowedProgram>,
allowed_program: Pubkey,
pub fn set_allowed_programs(
ctx: Context<UpdateWhitelist>,
allowed_programs: Vec<Pubkey>,
) -> Result<()> {
let whitelist = &mut ctx.accounts.whitelist;
require_keys_neq!(allowed_program, Pubkey::default());
require!(
!whitelist.allowed_programs.contains(&allowed_program),
AccumulatorUpdaterError::DuplicateAllowedProgram
);
whitelist.allowed_programs.push(allowed_program);
whitelist.validate_programs(&allowed_programs)?;
whitelist.allowed_programs = allowed_programs;
Ok(())
}
pub fn update_whitelist_authority(
ctx: Context<UpdateWhitelist>,
new_authority: Pubkey,
) -> Result<()> {
let whitelist = &mut ctx.accounts.whitelist;
whitelist.validate_new_authority(new_authority)?;
whitelist.authority = new_authority;
Ok(())
}
/// Add new account(s) to be included in the accumulator
///
/// * `base_account` - Pubkey of the original account the AccumulatorInput(s) are derived from
/// * `data` - Vec of AccumulatorInput account data
/// * `account_type` - Marker to indicate base_account account_type
/// * `account_schemas` - Vec of markers to indicate schemas for AccumulatorInputs. In same respective
/// order as data
/// * `base_account` - Pubkey of the original account the
/// AccumulatorInput(s) are derived from
/// * `data` - Vec of AccumulatorInput account data
/// * `account_type` - Marker to indicate base_account account_type
/// * `account_schemas` - Vec of markers to indicate schemas for
/// AccumulatorInputs. In same respective
/// order as data
pub fn create_inputs<'info>(
ctx: Context<'_, '_, '_, 'info, CreateInputs<'info>>,
base_account: Pubkey,
@ -105,10 +114,31 @@ pub mod accumulator_updater {
#[derive(InitSpace)]
pub struct Whitelist {
pub bump: u8,
pub authority: Pubkey,
#[max_len(32)]
pub allowed_programs: Vec<Pubkey>,
}
impl Whitelist {
pub fn validate_programs(&self, allowed_programs: &[Pubkey]) -> Result<()> {
require!(
!self.allowed_programs.contains(&Pubkey::default()),
AccumulatorUpdaterError::InvalidAllowedProgram
);
require_gte!(
32,
allowed_programs.len(),
AccumulatorUpdaterError::MaximumAllowedProgramsExceeded
);
Ok(())
}
pub fn validate_new_authority(&self, new_authority: Pubkey) -> Result<()> {
require_keys_neq!(new_authority, Pubkey::default());
Ok(())
}
}
#[derive(Accounts)]
pub struct WhitelistVerifier<'info> {
@ -141,7 +171,8 @@ impl<'info> WhitelistVerifier<'info> {
#[derive(Accounts)]
pub struct Initialize<'info> {
#[account(mut)]
pub payer: Signer<'info>,
pub payer: Signer<'info>,
#[account(
init,
payer = payer,
@ -153,17 +184,20 @@ pub struct Initialize<'info> {
pub system_program: Program<'info, System>,
}
#[derive(Accounts)]
pub struct AddAllowedProgram<'info> {
pub struct UpdateWhitelist<'info> {
#[account(mut)]
pub payer: Signer<'info>,
pub payer: Signer<'info>,
pub authority: Signer<'info>,
#[account(
mut,
seeds = [b"accumulator".as_ref(), b"whitelist".as_ref()],
bump = whitelist.bump,
mut,
seeds = [b"accumulator".as_ref(), b"whitelist".as_ref()],
bump = whitelist.bump,
has_one = authority
)]
pub whitelist: Account<'info, Whitelist>,
pub system_program: Program<'info, System>,
pub whitelist: Account<'info, Whitelist>,
}
@ -294,4 +328,10 @@ pub enum AccumulatorUpdaterError {
ConversionError,
#[msg("Serialization Error")]
SerializeError,
#[msg("Whitelist admin required on initialization")]
WhitelistAdminRequired,
#[msg("Invalid allowed program")]
InvalidAllowedProgram,
#[msg("Maximum number of allowed programs exceeded")]
MaximumAllowedProgramsExceeded,
}

View File

@ -18,3 +18,5 @@ default = []
[dependencies]
anchor-lang = "0.27.0"
accumulator_updater = { path = "../accumulator_updater", features = ["cpi"] }
# needed for the new #[account(zero_copy)] in anchor 0.27.0
bytemuck = { version = "1.4.0", features = ["derive", "min_const_generics"]}

View File

@ -10,8 +10,14 @@ use {
sysvar,
},
},
message::{
get_schemas,
price::*,
AccumulatorSerializer,
},
};
pub mod message;
declare_id!("Dg5PaFpoGXkYsidMpWTK6W2BeZ7FEfcYkg476zPFsLnS");
#[program]
@ -22,26 +28,27 @@ pub mod mock_cpi_caller {
ctx: Context<'_, '_, '_, 'info, AddPrice<'info>>,
params: AddPriceParams,
) -> Result<()> {
let pyth_price_acct = &mut ctx.accounts.pyth_price_account;
pyth_price_acct.init(params)?;
let mut account_data: Vec<Vec<u8>> = vec![];
let schemas = get_schemas(PythAccountType::Price);
let mut price_account_data_vec = vec![];
AccountSerialize::try_serialize(
&pyth_price_acct.clone().into_inner(),
&mut price_account_data_vec,
)?;
{
let pyth_price_acct = &mut ctx.accounts.pyth_price_account.load_init()?;
pyth_price_acct.init(params)?;
let price_full_data =
FullPriceMessage::from(&**pyth_price_acct).accumulator_serialize()?;
account_data.push(price_full_data);
let price_only_data = PriceOnly::from(&pyth_price_acct.clone().into_inner())
.try_to_vec()
.unwrap();
let price_compact_data =
CompactPriceMessage::from(&**pyth_price_acct).accumulator_serialize()?;
account_data.push(price_compact_data);
}
let account_data: Vec<Vec<u8>> = vec![price_account_data_vec, price_only_data];
let account_schemas = [PythSchemas::Full, PythSchemas::Compact]
.into_iter()
.map(|s| s.to_u8())
.collect::<Vec<u8>>();
let account_schemas = schemas.into_iter().map(|s| s.to_u8()).collect::<Vec<u8>>();
// 44444 compute units
// AddPrice::invoke_cpi_anchor(ctx, account_data, PythAccountType::Price, account_schemas)
@ -81,7 +88,6 @@ impl<'info> AddPrice<'info> {
account_schemas: Vec<u8>,
) -> Result<()> {
accumulator_updater::cpi::create_inputs(
// cpi_ctx,
ctx.accounts.create_inputs_ctx(ctx.remaining_accounts),
ctx.accounts.pyth_price_account.key(),
account_data,
@ -111,7 +117,7 @@ impl<'info> AddPrice<'info> {
.map(|a| AccountMeta::new(a.key(), false))
.collect::<Vec<_>>(),
);
let add_accumulator_input_ix = anchor_lang::solana_program::instruction::Instruction {
let create_inputs_ix = anchor_lang::solana_program::instruction::Instruction {
program_id: ctx.accounts.accumulator_program.key(),
accounts,
data: (
@ -127,7 +133,7 @@ impl<'info> AddPrice<'info> {
};
let account_infos = &mut ctx.accounts.to_account_infos();
account_infos.extend_from_slice(ctx.remaining_accounts);
anchor_lang::solana_program::program::invoke(&add_accumulator_input_ix, account_infos)?;
anchor_lang::solana_program::program::invoke(&create_inputs_ix, account_infos)?;
Ok(())
}
}
@ -153,6 +159,13 @@ pub struct AddPriceParams {
pub ema_expo: u64,
}
trait PythAccount {
const ACCOUNT_TYPE: PythAccountType;
fn account_type() -> PythAccountType {
Self::ACCOUNT_TYPE
}
}
#[derive(Copy, Clone)]
#[repr(u32)]
pub enum PythAccountType {
@ -168,20 +181,6 @@ impl PythAccountType {
}
}
#[derive(Copy, Clone)]
#[repr(u8)]
pub enum PythSchemas {
Full = 0,
Compact = 1,
Minimal = 2,
}
impl PythSchemas {
fn to_u8(&self) -> u8 {
*self as u8
}
}
#[derive(Accounts)]
#[instruction(params: AddPriceParams)]
pub struct AddPrice<'info> {
@ -192,7 +191,7 @@ pub struct AddPrice<'info> {
bump,
space = 8 + PriceAccount::INIT_SPACE
)]
pub pyth_price_account: Account<'info, PriceAccount>,
pub pyth_price_account: AccountLoader<'info, PriceAccount>,
#[account(mut)]
pub payer: Signer<'info>,
/// also needed for accumulator_updater
@ -208,8 +207,7 @@ pub struct AddPrice<'info> {
}
//Note: this will use anchor's default borsh serialization schema with the header
#[account]
#[account(zero_copy)]
#[derive(InitSpace)]
pub struct PriceAccount {
pub id: u64,
@ -217,6 +215,7 @@ pub struct PriceAccount {
pub price_expo: u64,
pub ema: u64,
pub ema_expo: u64,
pub comp_: [Pubkey; 32],
}
impl PriceAccount {
@ -230,45 +229,10 @@ impl PriceAccount {
}
}
// #[derive(Default, Debug, borsh::BorshSerialize)]
#[derive(AnchorSerialize, AnchorDeserialize, Default, Debug, Clone)]
pub struct PriceOnly {
pub price_expo: u64,
pub price: u64,
pub id: u64,
impl PythAccount for PriceAccount {
const ACCOUNT_TYPE: PythAccountType = PythAccountType::Price;
}
impl PriceOnly {
fn serialize(&self) -> Vec<u8> {
self.try_to_vec().unwrap()
}
fn serialize_from_price_account(other: PriceAccount) -> Vec<u8> {
PriceOnly::from(&other).try_to_vec().unwrap()
}
}
impl From<&PriceAccount> for PriceOnly {
fn from(other: &PriceAccount) -> Self {
Self {
id: other.id,
price: other.price,
price_expo: other.price_expo,
}
}
}
impl From<PriceAccount> for PriceOnly {
fn from(other: PriceAccount) -> Self {
Self {
id: other.id,
price: other.price,
price_expo: other.price_expo,
}
}
}
#[cfg(test)]
mod test {

View File

@ -0,0 +1,29 @@
use crate::PythAccountType;
pub mod price;
#[derive(Copy, Clone)]
#[repr(u8)]
pub enum MessageSchema {
Full = 0,
Compact = 1,
Minimal = 2,
}
impl MessageSchema {
pub fn to_u8(&self) -> u8 {
*self as u8
}
}
pub fn get_schemas(account_type: PythAccountType) -> Vec<MessageSchema> {
match account_type {
PythAccountType::Price => vec![MessageSchema::Full, MessageSchema::Compact],
_ => vec![MessageSchema::Full],
}
}
pub trait AccumulatorSerializer {
fn accumulator_serialize(&self) -> anchor_lang::Result<Vec<u8>>;
}

View File

@ -0,0 +1,78 @@
use {
crate::{
message::AccumulatorSerializer,
PriceAccount,
},
anchor_lang::prelude::*,
bytemuck::{
Pod,
Zeroable,
},
std::io::Write,
};
// TODO: should these schemas be "external" (protobuf?)
#[repr(C)]
#[derive(Debug, Copy, Clone, Pod, Zeroable)]
pub struct CompactPriceMessage {
pub price_expo: u64,
pub price: u64,
pub id: u64,
}
impl AccumulatorSerializer for CompactPriceMessage {
fn accumulator_serialize(&self) -> Result<Vec<u8>> {
let mut bytes = vec![];
bytes.write_all(&self.id.to_be_bytes())?;
bytes.write_all(&self.price.to_be_bytes())?;
bytes.write_all(&self.price_expo.to_be_bytes())?;
Ok(bytes)
}
}
impl From<&PriceAccount> for CompactPriceMessage {
fn from(other: &PriceAccount) -> Self {
Self {
id: other.id,
price: other.price,
price_expo: other.price_expo,
}
}
}
#[repr(C)]
#[derive(Debug, Copy, Clone, Pod, Zeroable)]
pub struct FullPriceMessage {
pub id: u64,
pub price: u64,
pub price_expo: u64,
pub ema: u64,
pub ema_expo: u64,
}
impl From<&PriceAccount> for FullPriceMessage {
fn from(other: &PriceAccount) -> Self {
Self {
id: other.id,
price: other.price,
price_expo: other.price_expo,
ema: other.ema,
ema_expo: other.ema_expo,
}
}
}
impl AccumulatorSerializer for FullPriceMessage {
fn accumulator_serialize(&self) -> Result<Vec<u8>> {
let mut bytes = vec![];
bytes.write_all(&self.id.to_be_bytes())?;
bytes.write_all(&self.price.to_be_bytes())?;
bytes.write_all(&self.price_expo.to_be_bytes())?;
bytes.write_all(&self.ema.to_be_bytes())?;
bytes.write_all(&self.ema_expo.to_be_bytes())?;
Ok(bytes)
}
}

View File

@ -1,25 +1,27 @@
import * as anchor from "@coral-xyz/anchor";
import { IdlTypes, Program, IdlAccounts } from "@coral-xyz/anchor";
import { IdlTypes, Program, BorshAccountsCoder } from "@coral-xyz/anchor";
import { AccumulatorUpdater } from "../target/types/accumulator_updater";
import { MockCpiCaller } from "../target/types/mock_cpi_caller";
import lumina from "@lumina-dev/test";
import { assert } from "chai";
import { ComputeBudgetProgram } from "@solana/web3.js";
import bs58 from "bs58";
// Enables tool that runs in localbrowser for easier debugging of txns
// in this test - https://lumina.fyi/debug
// Enables tool that runs in local browser for easier debugging of
// transactions in this test - https://lumina.fyi/debug
lumina();
const accumulatorUpdaterProgram = anchor.workspace
.AccumulatorUpdater as Program<AccumulatorUpdater>;
const mockCpiProg = anchor.workspace.MockCpiCaller as Program<MockCpiCaller>;
let whitelistAuthority = anchor.web3.Keypair.generate();
describe("accumulator_updater", () => {
// Configure the client to use the local cluster.
let provider = anchor.AnchorProvider.env();
anchor.setProvider(provider);
const [whitelistPda, whitelistBump] =
const [whitelistPubkey, whitelistBump] =
anchor.web3.PublicKey.findProgramAddressSync(
[Buffer.from("accumulator"), Buffer.from("whitelist")],
accumulatorUpdaterProgram.programId
@ -28,33 +30,57 @@ describe("accumulator_updater", () => {
it("Is initialized!", async () => {
// Add your test here.
const tx = await accumulatorUpdaterProgram.methods
.initialize()
.initialize(whitelistAuthority.publicKey)
.accounts({})
.rpc();
console.log("Your transaction signature", tx);
const whitelist = await accumulatorUpdaterProgram.account.whitelist.fetch(
whitelistPda
whitelistPubkey
);
assert.strictEqual(whitelist.bump, whitelistBump);
assert.isTrue(whitelist.authority.equals(whitelistAuthority.publicKey));
console.info(`whitelist: ${JSON.stringify(whitelist)}`);
});
it("Adds a program to the whitelist", async () => {
const addToWhitelistTx = await accumulatorUpdaterProgram.methods
.addAllowedProgram(mockCpiProg.programId)
.accounts({})
it("Sets allowed programs to the whitelist", async () => {
const allowedPrograms = [mockCpiProg.programId];
await accumulatorUpdaterProgram.methods
.setAllowedPrograms(allowedPrograms)
.accounts({
authority: whitelistAuthority.publicKey,
})
.signers([whitelistAuthority])
.rpc();
const whitelist = await accumulatorUpdaterProgram.account.whitelist.fetch(
whitelistPda
whitelistPubkey
);
console.info(`whitelist after add: ${JSON.stringify(whitelist)}`);
assert.isTrue(
whitelist.allowedPrograms
.map((pk) => pk.toString())
.includes(mockCpiProg.programId.toString())
const whitelistAllowedPrograms = whitelist.allowedPrograms.map((pk) =>
pk.toString()
);
assert.deepEqual(
whitelistAllowedPrograms,
allowedPrograms.map((p) => p.toString())
);
});
it("Updates the whitelist authority", async () => {
const newWhitelistAuthority = anchor.web3.Keypair.generate();
await accumulatorUpdaterProgram.methods
.updateWhitelistAuthority(newWhitelistAuthority.publicKey)
.accounts({
authority: whitelistAuthority.publicKey,
})
.signers([whitelistAuthority])
.rpc();
const whitelist = await accumulatorUpdaterProgram.account.whitelist.fetch(
whitelistPubkey
);
assert.isTrue(whitelist.authority.equals(newWhitelistAuthority.publicKey));
whitelistAuthority = newWhitelistAuthority;
});
it("Mock CPI program - AddPrice", async () => {
@ -71,7 +97,7 @@ describe("accumulator_updater", () => {
.accounts({
systemProgram: anchor.web3.SystemProgram.programId,
ixsSysvar: anchor.web3.SYSVAR_INSTRUCTIONS_PUBKEY,
accumulatorWhitelist: whitelistPda,
accumulatorWhitelist: whitelistPubkey,
accumulatorProgram: accumulatorUpdaterProgram.programId,
})
.pubkeys();
@ -136,16 +162,18 @@ describe("accumulator_updater", () => {
});
console.log(`addPriceTx: ${addPriceTx}`);
const accumulatorInputkeys = accumulatorPdas.map((a) => a.pubkey);
const pythPriceAccount = await provider.connection.getAccountInfo(
mockCpiCallerAddPriceTxPubkeys.pythPriceAccount
);
console.log(`pythPriceAccount: ${pythPriceAccount.data.toString("hex")}`);
const accumulatorInputKeys = accumulatorPdas.map((a) => a.pubkey);
const accumulatorInputs =
await accumulatorUpdaterProgram.account.accumulatorInput.fetchMultiple(
accumulatorInputkeys
accumulatorInputKeys
);
const accumulatorPriceAccounts = accumulatorInputs.map((ai) => {
const { header, data } = ai;
return parseAccumulatorInput(ai);
});
console.log(
@ -160,36 +188,93 @@ describe("accumulator_updater", () => {
assert.isTrue(pa.price.eq(addPriceParams.price));
assert.isTrue(pa.priceExpo.eq(addPriceParams.priceExpo));
});
let discriminator =
BorshAccountsCoder.accountDiscriminator("AccumulatorInput");
let accumulatorInputDiscriminator = bs58.encode(discriminator);
// fetch using `getProgramAccounts` and memcmp filter
const accumulatorAccounts = await provider.connection.getProgramAccounts(
accumulatorUpdaterProgram.programId,
{
filters: [
{
memcmp: {
offset: 0,
bytes: accumulatorInputDiscriminator,
},
},
],
}
);
const accumulatorInputKeyStrings = accumulatorInputKeys.map((k) =>
k.toString()
);
accumulatorAccounts.forEach((a) => {
assert.isTrue(accumulatorInputKeyStrings.includes(a.pubkey.toString()));
});
});
});
type AccumulatorInputHeader = IdlTypes<AccumulatorUpdater>["AccumulatorHeader"];
type AccumulatorInputPriceAccountTypes =
| IdlAccounts<MockCpiCaller>["priceAccount"] // case-sensitive
| IdlTypes<MockCpiCaller>["PriceOnly"];
// Parses AccumulatorInput.data into a PriceAccount or PriceOnly object based on the
// accountType and accountSchema.
//
// AccumulatorInput.data for AccumulatorInput<PriceAccount> will
// have mockCpiCaller::PriceAccount.discriminator()
// AccumulatorInput<PriceOnly> will not since its not an account
function parseAccumulatorInput({
header,
data,
}: {
header: AccumulatorInputHeader;
data: Buffer;
}): AccumulatorInputPriceAccountTypes {
}): AccumulatorPriceMessage {
console.log(`header: ${JSON.stringify(header)}`);
assert.strictEqual(header.accountType, 3);
if (header.accountSchema === 0) {
console.log(`[full]data: ${data.toString("hex")}`);
// case-sensitive. Note that "P" is capitalized here and not in
// the AccumulatorInputPriceAccountTypes type alias.
return mockCpiProg.coder.accounts.decode("PriceAccount", data);
return parseFullPriceMessage(data);
} else {
console.log(`[compact]data: ${data.toString("hex")}`);
return mockCpiProg.coder.types.decode("PriceOnly", data);
return parseCompactPriceMessage(data);
}
}
//TODO: follow wormhole sdk parsing structure?
// - https://github.com/wormhole-foundation/wormhole/blob/main/sdk/js/src/vaa/generic.ts
type AccumulatorPriceMessage = FullPriceMessage | CompactPriceMessage;
type FullPriceMessage = {
id: anchor.BN;
price: anchor.BN;
priceExpo: anchor.BN;
ema: anchor.BN;
emaExpo: anchor.BN;
};
function parseFullPriceMessage(data: Buffer): FullPriceMessage {
return {
id: new anchor.BN(data.subarray(0, 8), "be"),
price: new anchor.BN(data.subarray(8, 16), "be"),
priceExpo: new anchor.BN(data.subarray(16, 24), "be"),
ema: new anchor.BN(data.subarray(24, 32), "be"),
emaExpo: new anchor.BN(data.subarray(32, 40), "be"),
};
}
type CompactPriceMessage = {
id: anchor.BN;
price: anchor.BN;
priceExpo: anchor.BN;
};
function parseCompactPriceMessage(data: Buffer): CompactPriceMessage {
return {
id: new anchor.BN(data.subarray(0, 8), "be"),
price: new anchor.BN(data.subarray(8, 16), "be"),
priceExpo: new anchor.BN(data.subarray(16, 24), "be"),
};
}
interface AccumulatorInput<T> {
header: AccumulatorInputHeader;
account: T;
}