diff --git a/README.md b/README.md index 8bc2691..83672de 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,16 @@ yarn workspaces run build yarn workspace @switchboard-xyz/switchboardv2-cli link ``` +### Rust Setup + +The following command will build the anchor projects and update the program IDs + +``` +anchor build +node scripts/setup-example-programs +anchor test +``` + ### Python Setup ``` @@ -57,10 +67,13 @@ cd libraries/py poetry install ``` -### Build +### Localnet Testing Setup + +You may wish to run your own oracle for integration test. The following command will create a devnet Switchboard environment and output a `Switchboard.env` file to assist copying ``` -yarn workspaces run build +sbv2 localnet:env --keypair ../payer-keypair.json +chmod +x ./start-local-validator.sh && chmod +x ./start-oracle.sh ``` ## Test diff --git a/cli/README.md b/cli/README.md index 1502d32..1b8b184 100644 --- a/cli/README.md +++ b/cli/README.md @@ -169,11 +169,17 @@ OPTIONS --queueKey=queueKey (required) public key of the queue to create aggregator for + --sourceCluster=devnet|mainnet-beta alternative solana cluster to copy source aggregator from + --varianceThreshold=varianceThreshold override source aggregator's varianceThreshold -EXAMPLE - $ sbv2 aggregator:create:copy 8SXvChNYFhRq4EZuZvnhjrB3jJRQCv4k3P4W6hesH3Ee - AY3vpUu6v49shWajeFjHjgikYfaBWNJgax8zoEouUDTs --keypair ../payer-keypair.json +EXAMPLES + $ sbv2 aggregator:create:copy GvDMxPzN1sCj7L26YDK2HnMRXEQmQ2aemov8YBtPS7vR --queueKey + 9WZ59yz95bd3XwJxDPVE2PjvVWmSy9WM1NgGD2Hqsohw --keypair ../payer-keypair.json + $ sbv2 aggregator:create:copy GvDMxPzN1sCj7L26YDK2HnMRXEQmQ2aemov8YBtPS7vR --queueKey + 9WZ59yz95bd3XwJxDPVE2PjvVWmSy9WM1NgGD2Hqsohw --keypair ../payer-keypair.json --sourceCluster mainnet-beta + $ sbv2 aggregator:create:copy FcSmdsdWks75YdyCGegRqXdt5BiNGQKxZywyzb8ckD7D --queueKey + 9WZ59yz95bd3XwJxDPVE2PjvVWmSy9WM1NgGD2Hqsohw --keypair ../payer-keypair.json --sourceCluster mainnet-beta ``` _See code: [src/commands/aggregator/create/copy.ts](https://github.com/switchboard-xyz/switchboard-v2/blob/v0.1.18/src/commands/aggregator/create/copy.ts)_ @@ -1302,22 +1308,24 @@ USAGE $ sbv2 localnet:env OPTIONS - -h, --help show CLI help + -h, --help show CLI help - -k, --keypair=keypair keypair that will pay for onchain transactions. defaults to new account authority if no - alternate authority provided + -k, --keypair=keypair keypair that will pay for onchain transactions. defaults to new account authority if no + alternate authority provided - -s, --silent suppress cli prompts + -o, --outputDir=outputDir output directory for scripts - -u, --rpcUrl=rpcUrl alternate RPC url + -s, --silent suppress cli prompts - -v, --verbose log everything + -u, --rpcUrl=rpcUrl alternate RPC url - --force overwrite output file if existing + -v, --verbose log everything - --mainnetBeta WARNING: use mainnet-beta solana cluster + --force overwrite output file if existing - --programId=programId alternative Switchboard program ID to interact with + --mainnetBeta WARNING: use mainnet-beta solana cluster + + --programId=programId alternative Switchboard program ID to interact with ``` _See code: [src/commands/localnet/env.ts](https://github.com/switchboard-xyz/switchboard-v2/blob/v0.1.18/src/commands/localnet/env.ts)_ @@ -1635,22 +1643,24 @@ ARGUMENTS AGGREGATORKEY public key of the aggregator account to deserialize OPTIONS - -h, --help show CLI help + -h, --help show CLI help - -k, --keypair=keypair keypair that will pay for onchain transactions. defaults to new account authority if no - alternate authority provided + -k, --keypair=keypair keypair that will pay for onchain transactions. defaults to new account authority if no + alternate authority provided - -s, --silent suppress cli prompts + -o, --oraclePubkeysData print the assigned oracles for the current round - -u, --rpcUrl=rpcUrl alternate RPC url + -s, --silent suppress cli prompts - -v, --verbose log everything + -u, --rpcUrl=rpcUrl alternate RPC url - --jobs output job definitions + -v, --verbose log everything - --mainnetBeta WARNING: use mainnet-beta solana cluster + --jobs output job definitions - --programId=programId alternative Switchboard program ID to interact with + --mainnetBeta WARNING: use mainnet-beta solana cluster + + --programId=programId alternative Switchboard program ID to interact with ALIASES $ sbv2 aggregator:print @@ -2145,6 +2155,8 @@ OPTIONS -v, --verbose log everything + --enableBufferRelayers enable oracles to fulfill buffer relayer requests + --force overwrite output file if existing --mainnetBeta WARNING: use mainnet-beta solana cluster @@ -2157,6 +2169,8 @@ OPTIONS --unpermissionedFeeds permit unpermissioned feeds + --unpermissionedVrf permit unpermissioned VRF accounts + ALIASES $ sbv2 custom:queue ``` diff --git a/cli/package.json b/cli/package.json index 8f5ce8f..7db52b8 100644 --- a/cli/package.json +++ b/cli/package.json @@ -98,8 +98,8 @@ "@oclif/plugin-warn-if-update-available": "^1.7.3", "@project-serum/anchor": "^0.24.2", "@solana/spl-token": "^0.1.8", - "@solana/web3.js": "^1.41.10", - "@switchboard-xyz/sbv2-utils": "^0.0.9", + "@solana/web3.js": "1.39.1", + "@switchboard-xyz/sbv2-utils": "^0.0.10", "@switchboard-xyz/switchboard-v2": "0.0.97", "assert": "^2.0.0", "big.js": "^6.1.1", diff --git a/cli/src/commands/aggregator/create/copy.ts b/cli/src/commands/aggregator/create/copy.ts index e3bdcf8..30ba9cc 100644 --- a/cli/src/commands/aggregator/create/copy.ts +++ b/cli/src/commands/aggregator/create/copy.ts @@ -17,6 +17,7 @@ import { CrankAccount, JobAccount, LeaseAccount, + loadSwitchboardProgram, OracleJob, OracleQueueAccount, PermissionAccount, @@ -69,6 +70,11 @@ export default class AggregatorCreateCopy extends BaseCommand { description: "public key of the crank to push aggregator to", required: false, }), + sourceCluster: flags.string({ + description: "alternative solana cluster to copy source aggregator from", + required: false, + options: ["devnet", "mainnet-beta"], + }), }; static args = [ @@ -81,7 +87,9 @@ export default class AggregatorCreateCopy extends BaseCommand { ]; static examples = [ - "$ sbv2 aggregator:create:copy 8SXvChNYFhRq4EZuZvnhjrB3jJRQCv4k3P4W6hesH3Ee AY3vpUu6v49shWajeFjHjgikYfaBWNJgax8zoEouUDTs --keypair ../payer-keypair.json", + "$ sbv2 aggregator:create:copy GvDMxPzN1sCj7L26YDK2HnMRXEQmQ2aemov8YBtPS7vR --queueKey 9WZ59yz95bd3XwJxDPVE2PjvVWmSy9WM1NgGD2Hqsohw --keypair ../payer-keypair.json", + "$ sbv2 aggregator:create:copy GvDMxPzN1sCj7L26YDK2HnMRXEQmQ2aemov8YBtPS7vR --queueKey 9WZ59yz95bd3XwJxDPVE2PjvVWmSy9WM1NgGD2Hqsohw --keypair ../payer-keypair.json --sourceCluster mainnet-beta", + "$ sbv2 aggregator:create:copy FcSmdsdWks75YdyCGegRqXdt5BiNGQKxZywyzb8ckD7D --queueKey 9WZ59yz95bd3XwJxDPVE2PjvVWmSy9WM1NgGD2Hqsohw --keypair ../payer-keypair.json --sourceCluster mainnet-beta", ]; async run() { @@ -90,6 +98,42 @@ export default class AggregatorCreateCopy extends BaseCommand { const payerKeypair = programWallet(this.program); + const sourceProgram = !flags.sourceCluster + ? this.program + : flags.sourceCluster === "devnet" || + flags.sourceCluster === "mainnet-beta" + ? await loadSwitchboardProgram( + flags.sourceCluster, + undefined, + payerKeypair + ) + : undefined; + if (sourceProgram === undefined) { + throw new Error(`Invalid sourceAggregatorCluster ${flags.sourceCluster}`); + } + const sourceAggregatorAccount = new AggregatorAccount({ + program: sourceProgram, + publicKey: args.aggregatorSource, + }); + + const sourceAggregator = await sourceAggregatorAccount.loadData(); + const sourceJobPubkeys: PublicKey[] = sourceAggregator.jobPubkeysData.slice( + 0, + sourceAggregator.jobPubkeysSize + ); + + const sourceJobAccounts = sourceJobPubkeys.map((publicKey) => { + return new JobAccount({ program: sourceProgram, publicKey: publicKey }); + }); + + const sourceJobs = await Promise.all( + sourceJobAccounts.map(async (jobAccount) => { + const data = await jobAccount.loadData(); + const job = OracleJob.decodeDelimited(data.data); + return { job, data }; + }) + ); + const [programStateAccount, stateBump] = ProgramStateAccount.fromSeed( this.program ); @@ -105,28 +149,6 @@ export default class AggregatorCreateCopy extends BaseCommand { await tokenMint.getOrCreateAssociatedAccountInfo(payerKeypair.publicKey) ).address; - const sourceAggregatorAccount = new AggregatorAccount({ - program: this.program, - publicKey: args.aggregatorSource, - }); - const sourceAggregator = await sourceAggregatorAccount.loadData(); - const sourceJobPubkeys: PublicKey[] = sourceAggregator.jobPubkeysData.slice( - 0, - sourceAggregator.jobPubkeysSize - ); - - const sourceJobAccounts = sourceJobPubkeys.map((publicKey) => { - return new JobAccount({ program: this.program, publicKey: publicKey }); - }); - - const sourceJobs = await Promise.all( - sourceJobAccounts.map(async (jobAccount) => { - const data = await jobAccount.loadData(); - const job = OracleJob.decodeDelimited(data.data); - return { job, data }; - }) - ); - const createAccountInstructions: ( | TransactionInstruction | TransactionInstruction[] diff --git a/cli/src/commands/localnet/env.ts b/cli/src/commands/localnet/env.ts index 01163ee..914c8e9 100644 --- a/cli/src/commands/localnet/env.ts +++ b/cli/src/commands/localnet/env.ts @@ -18,6 +18,10 @@ export default class LocalnetEnvironment extends BaseCommand { description: "overwrite output file if existing", default: false, }), + outputDir: flags.string({ + char: "o", + description: "output directory for scripts", + }), }; async run() { @@ -25,24 +29,28 @@ export default class LocalnetEnvironment extends BaseCommand { const { flags } = this.parse(LocalnetEnvironment); const payerKeypair = programWallet(this.program); + const outputDir = flags.outputDir + ? path.join(process.cwd(), flags.outputDir) + : process.cwd(); + // TODO: Check paths and force flags if (!flags.force) { - if (fs.existsSync(path.join(process.cwd(), "switchboard.env"))) { + if (fs.existsSync(path.join(outputDir, "switchboard.env"))) { throw new Error( "switchboard.env already exists, use --force to overwrite" ); } - if (fs.existsSync(path.join(process.cwd(), "switchboard.json"))) { + if (fs.existsSync(path.join(outputDir, "switchboard.json"))) { throw new Error( "switchboard.json already exists, use --force to overwrite" ); } - if (fs.existsSync(path.join(process.cwd(), "start-local-validator.sh"))) { + if (fs.existsSync(path.join(outputDir, "start-local-validator.sh"))) { throw new Error( "start-local-validator.sh already exists, use --force to overwrite" ); } - if (fs.existsSync(path.join(process.cwd(), "start-oracle.sh"))) { + if (fs.existsSync(path.join(outputDir, "start-oracle.sh"))) { throw new Error( "start-oracle.sh already exists, use --force to overwrite" ); @@ -70,7 +78,8 @@ export default class LocalnetEnvironment extends BaseCommand { new PublicKey(flags.programId) ); // TODO: Add silent flag - testEnvironment.writeAll(flags.keypair, process.cwd()); + fs.mkdirSync(outputDir, { recursive: true }); + testEnvironment.writeAll(flags.keypair, outputDir); } async catch(error) { diff --git a/cli/src/commands/print/aggregator.ts b/cli/src/commands/print/aggregator.ts index fd1f7d9..0415bb9 100644 --- a/cli/src/commands/print/aggregator.ts +++ b/cli/src/commands/print/aggregator.ts @@ -1,7 +1,10 @@ /* eslint-disable unicorn/import-style */ import { flags } from "@oclif/command"; import { PublicKey } from "@solana/web3.js"; -import { prettyPrintAggregator } from "@switchboard-xyz/sbv2-utils"; +import { + chalkString, + prettyPrintAggregator, +} from "@switchboard-xyz/sbv2-utils"; import { AggregatorAccount } from "@switchboard-xyz/switchboard-v2"; import BaseCommand from "../../BaseCommand"; @@ -16,6 +19,10 @@ export default class AggregatorPrint extends BaseCommand { description: "output job definitions", default: false, }), + oraclePubkeysData: flags.boolean({ + char: "o", + description: "print the assigned oracles for the current round", + }), }; static args = [ @@ -49,6 +56,19 @@ export default class AggregatorPrint extends BaseCommand { flags.jobs ) ); + + if (flags.oraclePubkeysData) { + this.logger.log( + chalkString( + "oraclePubkeyData", + "\n" + + (aggregator.currentRound.oraclePubkeysData as PublicKey[]) + .filter((pubkey) => !PublicKey.default.equals(pubkey)) + .map((pubkey) => pubkey.toString()) + .join("\n") + ) + ); + } } async catch(error) { diff --git a/cli/src/commands/queue/create.ts b/cli/src/commands/queue/create.ts index 13a9f82..5bafa0b 100644 --- a/cli/src/commands/queue/create.ts +++ b/cli/src/commands/queue/create.ts @@ -4,17 +4,16 @@ import { flags } from "@oclif/command"; import * as anchor from "@project-serum/anchor"; import * as spl from "@solana/spl-token"; import { - AccountInfo, Keypair, SystemProgram, TransactionInstruction, } from "@solana/web3.js"; import { chalkString, + packAndSend, prettyPrintCrank, prettyPrintOracle, prettyPrintQueue, - promiseWithTimeout, } from "@switchboard-xyz/sbv2-utils"; import { CrankAccount, @@ -30,7 +29,6 @@ import fs from "fs"; import path from "path"; import BaseCommand from "../../BaseCommand"; import { sleep, verifyProgramHasPayer } from "../../utils"; -import { packAndSend } from "../../utils/transaction"; export default class QueueCreate extends BaseCommand { static description = "create a custom queue"; @@ -85,6 +83,14 @@ export default class QueueCreate extends BaseCommand { description: "permit unpermissioned feeds", default: false, }), + unpermissionedVrf: flags.boolean({ + description: "permit unpermissioned VRF accounts", + default: false, + }), + enableBufferRelayers: flags.boolean({ + description: "enable oracles to fulfill buffer relayer requests", + default: false, + }), outputFile: flags.string({ char: "f", description: "output queue schema to a json file", @@ -96,6 +102,7 @@ export default class QueueCreate extends BaseCommand { verifyProgramHasPayer(this.program); const { flags, args } = this.parse(QueueCreate); const payerKeypair = programWallet(this.program); + const signers: Keypair[] = []; const outputPath = flags.outputFile === undefined @@ -109,6 +116,9 @@ export default class QueueCreate extends BaseCommand { } const authorityKeypair = await this.loadAuthority(flags.authority); + if (!authorityKeypair.publicKey.equals(payerKeypair.publicKey)) { + signers.push(authorityKeypair); + } const [programStateAccount, stateBump] = ProgramStateAccount.fromSeed( this.program ); @@ -120,7 +130,7 @@ export default class QueueCreate extends BaseCommand { ); const ixns: (TransactionInstruction | TransactionInstruction[])[] = []; - const signers: Keypair[] = [payerKeypair, authorityKeypair]; + // const signers: Keypair[] = [payerKeypair, authorityKeypair]; try { await programStateAccount.loadData(); @@ -215,6 +225,8 @@ export default class QueueCreate extends BaseCommand { minimumDelaySeconds: 5, queueSize: flags.queueSize, unpermissionedFeeds: flags.unpermissionedFeeds ?? false, + unpermissionedVrf: flags.unpermissionedVrf ?? false, + enableBufferRelayers: flags.enableBufferRelayers ?? false, }) .accounts({ oracleQueue: queueKeypair.publicKey, @@ -345,35 +357,16 @@ export default class QueueCreate extends BaseCommand { }) ); - const createAccountSignatures = packAndSend( + // console.log(`${signers.map((s) => s.publicKey.toString()).join("\n")}`); + + const createAccountSignatures = await packAndSend( this.program, - ixns, - finalTransactions, + [ixns, finalTransactions], signers, payerKeypair.publicKey ); - let queueWs: number; - const customQueuePromise = new Promise((resolve: (result: any) => void) => { - queueWs = this.program.provider.connection.onAccountChange( - queueAccount.publicKey, - (accountInfo: AccountInfo, slot) => { - const accountCoder = new anchor.BorshAccountsCoder(this.program.idl); - resolve( - accountCoder.decode("OracleQueueAccountData", accountInfo.data) - ); - } - ); - }); - - const queueData = await promiseWithTimeout( - 22_000, - customQueuePromise - ).finally(() => { - try { - this.program.provider.connection.removeAccountChangeListener(queueWs); - } catch {} - }); + const queueData = await queueAccount.loadData(); if (outputPath) { fs.mkdirSync(path.dirname(outputPath), { recursive: true }); diff --git a/cli/src/utils/transaction.ts b/cli/src/utils/transaction.ts index bd0e798..8c7c42e 100644 --- a/cli/src/utils/transaction.ts +++ b/cli/src/utils/transaction.ts @@ -22,15 +22,14 @@ export async function packAndSend( const packedTransactions = packInstructions(ixnsBatch, feePayer, blockhash); const signedTransactions = signTransactions(packedTransactions, signers); - const signedTxs = await ( - program.provider as anchor.AnchorProvider - ).wallet.signAllTransactions(signedTransactions); + // const signedTxs = await ( + // program.provider as anchor.AnchorProvider + // ).wallet.signAllTransactions(signedTransactions); for (let k = 0; k < packedTransactions.length; k += 1) { - const tx = signedTxs[k]; - const rawTx = tx.serialize(); + const tx = signedTransactions[k]; signatures.push( - program.provider.connection.sendRawTransaction(rawTx, { + program.provider.connection.sendTransaction(tx, signers, { skipPreflight: true, maxRetries: 10, }) diff --git a/libraries/sbv2-utils/package.json b/libraries/sbv2-utils/package.json index c804447..5566178 100644 --- a/libraries/sbv2-utils/package.json +++ b/libraries/sbv2-utils/package.json @@ -34,7 +34,7 @@ "@project-serum/anchor": "^0.24.2", "@saberhq/token-utils": "^1.12.68", "@solana/spl-token": "^0.1.8", - "@solana/web3.js": "^1.37.1", + "@solana/web3.js": "1.39.1", "@switchboard-xyz/switchboard-v2": "^0.0.97", "big.js": "^6.1.1", "chalk": "4", diff --git a/libraries/sbv2-utils/src/feeds.ts b/libraries/sbv2-utils/src/feeds.ts deleted file mode 100644 index c45e419..0000000 --- a/libraries/sbv2-utils/src/feeds.ts +++ /dev/null @@ -1,326 +0,0 @@ -import * as anchor from "@project-serum/anchor"; -import * as spl from "@solana/spl-token"; -import { - Keypair, - PublicKey, - SystemProgram, - TransactionInstruction, -} from "@solana/web3.js"; -import { - AggregatorAccount, - CrankAccount, - JobAccount, - LeaseAccount, - OracleJob, - OracleQueueAccount, - PermissionAccount, - ProgramStateAccount, - SwitchboardDecimal, -} from "@switchboard-xyz/switchboard-v2"; -import type Big from "big.js"; - -interface CopyAggregatorParameters { - authority?: PublicKey; - minOracles?: number; - batchSize?: number; - minJobs?: number; - minUpdateDelay?: number; - forceReportPeriod?: number; - varianceThreshold?: Big; - crankKey?: PublicKey; -} - -export async function copyAggregatorTxn( - payerKeypair: Keypair, - sourceAggregatorAccount: AggregatorAccount, - targetQueue: OracleQueueAccount, - params: CopyAggregatorParameters -) { - // load source environment - const sourceAggregator = await sourceAggregatorAccount.loadData(); - const sourceJobPubkeys: PublicKey[] = sourceAggregator.jobPubkeysData.slice( - 0, - sourceAggregator.jobPubkeysSize - ); - const sourceJobAccounts = sourceJobPubkeys.map((publicKey) => { - return new JobAccount({ - program: sourceAggregatorAccount.program, - publicKey: publicKey, - }); - }); - const sourceJobs = await Promise.all( - sourceJobAccounts.map(async (jobAccount) => { - const data = await jobAccount.loadData(); - const job = OracleJob.decodeDelimited(data.data); - return { job, data }; - }) - ); - - const program = targetQueue.program; - - const [programStateAccount, stateBump] = - ProgramStateAccount.fromSeed(program); - const programState = await programStateAccount.loadData(); - const queue = await targetQueue.loadData(); - - const tokenMint = await targetQueue.loadMint(); - const tokenWallet = ( - await tokenMint.getOrCreateAssociatedAccountInfo(payerKeypair.publicKey) - ).address; - - const createAccountInstructions: ( - | TransactionInstruction - | TransactionInstruction[] - )[] = []; - const createAccountSigners: Keypair[] = [payerKeypair]; - - const jobAccounts = await Promise.all( - sourceJobs.map(async ({ job, data }) => { - const jobKeypair = Keypair.generate(); - createAccountSigners.push(jobKeypair); - - const jobData = Buffer.from( - OracleJob.encodeDelimited( - OracleJob.create({ - tasks: job.tasks, - }) - ).finish() - ); - const size = - 280 + jobData.length + (data.variables?.join("")?.length ?? 0); - - createAccountInstructions.push([ - SystemProgram.createAccount({ - fromPubkey: payerKeypair.publicKey, - newAccountPubkey: jobKeypair.publicKey, - space: size, - lamports: - await this.program.provider.connection.getMinimumBalanceForRentExemption( - size - ), - programId: this.program.programId, - }), - await this.program.methods - .jobInit({ - name: Buffer.from(data.name), - data: jobData, - variables: - data.variables?.map((item) => Buffer.from("")) ?? - new Array(), - authorWallet: payerKeypair.publicKey, - stateBump, - }) - .accounts({ - job: jobKeypair.publicKey, - authorWallet: tokenWallet, - authority: payerKeypair.publicKey, - programState: programStateAccount.publicKey, - }) - // .signers([jobKeypair]) - .instruction(), - ]); - - return new JobAccount({ - program: this.program, - publicKey: jobKeypair.publicKey, - }); - }) - ); - - const aggregatorKeypair = Keypair.generate(); - this.logger.debug(`Aggregator: ${aggregatorKeypair.publicKey}`); - createAccountSigners.push(aggregatorKeypair); - const aggregatorSize = this.program.account.aggregatorAccountData.size; - const permissionAccountSize = this.program.account.permissionAccountData.size; - const [permissionAccount, permissionBump] = PermissionAccount.fromSeed( - this.program, - queue.authority, - targetQueue.publicKey, - aggregatorKeypair.publicKey - ); - - const aggregatorAccount = new AggregatorAccount({ - program: this.program, - publicKey: aggregatorKeypair.publicKey, - }); - - // Create lease and push to crank - const [leaseAccount, leaseBump] = LeaseAccount.fromSeed( - this.program, - targetQueue, - aggregatorAccount - ); - const leaseEscrow = await spl.Token.getAssociatedTokenAddress( - spl.ASSOCIATED_TOKEN_PROGRAM_ID, - spl.TOKEN_PROGRAM_ID, - tokenMint.publicKey, - leaseAccount.publicKey, - true - ); - - const jobPubkeys: Array = []; - const jobWallets: Array = []; - const walletBumps: Array = []; - for (const idx in jobAccounts) { - const [jobWallet, bump] = anchor.utils.publicKey.findProgramAddressSync( - [ - payerKeypair.publicKey.toBuffer(), - spl.TOKEN_PROGRAM_ID.toBuffer(), - tokenMint.publicKey.toBuffer(), - ], - spl.ASSOCIATED_TOKEN_PROGRAM_ID - ); - jobPubkeys.push(jobAccounts[idx].publicKey); - jobWallets.push(jobWallet); - walletBumps.push(bump); - } - - createAccountInstructions.push( - [ - // allocate aggregator space - SystemProgram.createAccount({ - fromPubkey: payerKeypair.publicKey, - newAccountPubkey: aggregatorKeypair.publicKey, - space: aggregatorSize, - lamports: - await this.program.provider.connection.getMinimumBalanceForRentExemption( - aggregatorSize - ), - programId: this.program.programId, - }), - // create aggregator - await this.program.methods - .aggregatorInit({ - name: sourceAggregator.name, - metadata: sourceAggregator.metadata, - batchSize: - params.batchSize ?? sourceAggregator.oracleRequestBatchSize, - minOracleResults: - params.minOracles ?? sourceAggregator.minOracleResults, - minJobResults: params.minJobs ?? sourceAggregator.minJobResults, - minUpdateDelaySeconds: - params.minUpdateDelay ?? sourceAggregator.minUpdateDelaySeconds, - varianceThreshold: params.varianceThreshold - ? SwitchboardDecimal.fromBig(params.varianceThreshold) - : sourceAggregator.varianceThreshold, - forceReportPeriod: - params.forceReportPeriod ?? sourceAggregator.forceReportPeriod, - stateBump, - }) - .accounts({ - aggregator: aggregatorKeypair.publicKey, - authority: payerKeypair.publicKey, - queue: targetQueue.publicKey, - authorWallet: tokenWallet, - programState: programStateAccount.publicKey, - }) - .instruction(), - // create permissions - await this.program.methods - .permissionInit({}) - .accounts({ - permission: permissionAccount.publicKey, - authority: queue.authority, - granter: targetQueue.publicKey, - grantee: aggregatorKeypair.publicKey, - payer: payerKeypair.publicKey, - systemProgram: SystemProgram.programId, - }) - .instruction(), - payerKeypair.publicKey.equals(queue.authority) - ? await this.program.methods - .permissionSet({ - permission: { permitOracleQueueUsage: null }, - enable: true, - }) - .accounts({ - permission: permissionAccount.publicKey, - authority: queue.authority, - }) - .instruction() - : undefined, - spl.Token.createAssociatedTokenAccountInstruction( - spl.ASSOCIATED_TOKEN_PROGRAM_ID, - spl.TOKEN_PROGRAM_ID, - tokenMint.publicKey, - leaseEscrow, - leaseAccount.publicKey, - payerKeypair.publicKey - ), - await this.program.methods - .leaseInit({ - loadAmount: new anchor.BN(0), - stateBump, - leaseBump, - withdrawAuthority: payerKeypair.publicKey, - walletBumps: Buffer.from([]), - }) - .accounts({ - programState: programStateAccount.publicKey, - lease: leaseAccount.publicKey, - queue: targetQueue.publicKey, - aggregator: aggregatorAccount.publicKey, - systemProgram: SystemProgram.programId, - funder: tokenWallet, - payer: payerKeypair.publicKey, - tokenProgram: spl.TOKEN_PROGRAM_ID, - escrow: leaseEscrow, - owner: payerKeypair.publicKey, - mint: tokenMint.publicKey, - }) - // .remainingAccounts( - // jobPubkeys.concat(jobWallets).map((pubkey: PublicKey) => { - // return { isSigner: false, isWritable: true, pubkey }; - // }) - // ) - .instruction(), - params.crankKey - ? await this.program.methods - .crankPush({ - stateBump, - permissionBump, - }) - .accounts({ - crank: new PublicKey(params.crankKey), - aggregator: aggregatorAccount.publicKey, - oracleQueue: targetQueue.publicKey, - queueAuthority: queue.authority, - permission: permissionAccount.publicKey, - lease: leaseAccount.publicKey, - escrow: leaseEscrow, - programState: programStateAccount.publicKey, - dataBuffer: ( - await new CrankAccount({ - program: this.program, - publicKey: new PublicKey(params.crankKey), - }).loadData() - ).dataBuffer, - }) - .instruction() - : undefined, - ].filter((item) => item) - ); - - const finalInstructions: ( - | TransactionInstruction - | TransactionInstruction[] - )[] = []; - - finalInstructions.push( - ...(await Promise.all( - jobAccounts.map(async (jobAccount) => { - return this.program.methods - .aggregatorAddJob({ - weight: 1, - }) - .accounts({ - aggregator: aggregatorKeypair.publicKey, - authority: payerKeypair.publicKey, - job: jobAccount.publicKey, - }) - .instruction(); - }) - )) - ); - return ""; -} diff --git a/libraries/sbv2-utils/src/index.ts b/libraries/sbv2-utils/src/index.ts index 89286e0..dfbeafb 100644 --- a/libraries/sbv2-utils/src/index.ts +++ b/libraries/sbv2-utils/src/index.ts @@ -6,7 +6,7 @@ export * from "./date"; export * from "./errors"; export * from "./nonce"; export * from "./print"; -export * from "./solana"; export * from "./state"; export * from "./switchboard"; export * from "./test"; +export * from "./transaction"; diff --git a/libraries/sbv2-utils/src/print.ts b/libraries/sbv2-utils/src/print.ts index 87304b9..44aea65 100644 --- a/libraries/sbv2-utils/src/print.ts +++ b/libraries/sbv2-utils/src/print.ts @@ -284,11 +284,16 @@ export async function prettyPrintQueue( outputString += chalk.underline( chalkString("\r\n## Oracles", " ".repeat(32), SPACING) + "\r\n" ); - data.queue.forEach( - (row: PublicKey, index) => - (outputString += - chalkString(`# ${index + 1}`, row.toString(), SPACING) + "\r\n") - ); + outputString += (data.queue as PublicKey[]) + .filter((pubkey) => !PublicKey.default.equals(pubkey)) + .map((pubkey) => pubkey.toString()) + .join("\n"); + + // (data.queue as PublicKey[]).forEach( + // (row, index) => + // (outputString += + // chalkString(`# ${index + 1},`, row.toString(), SPACING) + "\r\n") + // ); } return outputString; diff --git a/libraries/sbv2-utils/src/queue.ts b/libraries/sbv2-utils/src/queue.ts new file mode 100644 index 0000000..1b3e36c --- /dev/null +++ b/libraries/sbv2-utils/src/queue.ts @@ -0,0 +1,302 @@ +import * as anchor from "@project-serum/anchor"; +import * as spl from "@solana/spl-token"; +import { + Keypair, + PublicKey, + SystemProgram, + TransactionInstruction, +} from "@solana/web3.js"; +import { + CrankAccount, + OracleAccount, + OracleQueueAccount, + PermissionAccount, + ProgramStateAccount, + programWallet, + SwitchboardDecimal, +} from "@switchboard-xyz/switchboard-v2"; +import Big from "big.js"; +import { chalkString } from "./print"; +import { packAndSend } from "./transaction"; + +export interface CreateQueueParams { + authority?: PublicKey; + name?: string; + metadata?: string; + minStake: anchor.BN; + reward: anchor.BN; + crankSize?: number; + oracleTimeout?: number; + numOracles?: number; + unpermissionedFeeds?: boolean; + unpermissionedVrf?: boolean; +} + +export interface CreateQueueResponse { + queueAccount: OracleQueueAccount; + crankPubkey: PublicKey; + oracles: PublicKey[]; +} + +export async function createQueue( + program: anchor.Program, + params: CreateQueueParams, + queueSize = 500, + authorityKeypair = programWallet(program) +): Promise { + const payerKeypair = programWallet(program); + + const [programStateAccount, stateBump] = ProgramStateAccount.fromSeed( + this.program + ); + const tokenMint = new spl.Token( + this.program.provider.connection, + spl.NATIVE_MINT, + spl.TOKEN_PROGRAM_ID, + payerKeypair + ); + + const ixns: (TransactionInstruction | TransactionInstruction[])[] = []; + const signers: Keypair[] = [payerKeypair, authorityKeypair]; + + try { + await programStateAccount.loadData(); + } catch { + const vaultKeypair = anchor.web3.Keypair.generate(); + ixns.push([ + SystemProgram.createAccount({ + fromPubkey: payerKeypair.publicKey, + newAccountPubkey: vaultKeypair.publicKey, + lamports: + await this.program.provider.connection.getMinimumBalanceForRentExemption( + spl.AccountLayout.span + ), + space: spl.AccountLayout.span, + programId: spl.TOKEN_PROGRAM_ID, + }), + spl.Token.createInitAccountInstruction( + spl.TOKEN_PROGRAM_ID, + tokenMint.publicKey, + vaultKeypair.publicKey, + payerKeypair.publicKey + ), + await this.program.methods + .programInit({ + stateBump, + }) + .accounts({ + state: programStateAccount.publicKey, + authority: payerKeypair.publicKey, + tokenMint: tokenMint.publicKey, + vault: vaultKeypair.publicKey, + payer: payerKeypair.publicKey, + systemProgram: SystemProgram.programId, + tokenProgram: spl.TOKEN_PROGRAM_ID, + daoMint: tokenMint.publicKey, + }) + .instruction(), + ]); + signers.push(vaultKeypair); + } + + const queueKeypair = anchor.web3.Keypair.generate(); + const queueBuffer = anchor.web3.Keypair.generate(); + const queueBufferSize = queueSize * 32 + 8; + + const queueAccount = new OracleQueueAccount({ + program: this.program, + publicKey: queueKeypair.publicKey, + }); + + this.logger.debug(chalkString("OracleQueue", queueKeypair.publicKey)); + this.logger.debug(chalkString("OracleBuffer", queueBuffer.publicKey)); + + const crankKeypair = anchor.web3.Keypair.generate(); + const crankBuffer = anchor.web3.Keypair.generate(); + const crankSize = params.crankSize ? params.crankSize * 40 + 8 : 0; + + this.logger.debug(chalkString("CrankAccount", crankKeypair.publicKey)); + this.logger.debug(chalkString("CrankBuffer", crankBuffer.publicKey)); + + const crankAccount = new CrankAccount({ + program: this.program, + publicKey: crankKeypair.publicKey, + }); + + ixns.push( + anchor.web3.SystemProgram.createAccount({ + fromPubkey: payerKeypair.publicKey, + newAccountPubkey: queueBuffer.publicKey, + space: queueBufferSize, + lamports: + await this.program.provider.connection.getMinimumBalanceForRentExemption( + queueBufferSize + ), + programId: this.program.programId, + }), + await this.program.methods + .oracleQueueInit({ + name: Buffer.from(params.name).slice(0, 32), + metadata: Buffer.from("").slice(0, 64), + reward: params.reward ? new anchor.BN(params.reward) : new anchor.BN(0), + minStake: params.minStake + ? new anchor.BN(params.minStake) + : new anchor.BN(0), + // feedProbationPeriod: 0, + oracleTimeout: params.oracleTimeout, + slashingEnabled: false, + varianceToleranceMultiplier: SwitchboardDecimal.fromBig(new Big(2)), + authority: authorityKeypair.publicKey, + // consecutiveFeedFailureLimit: new anchor.BN(1000), + // consecutiveOracleFailureLimit: new anchor.BN(1000), + minimumDelaySeconds: 5, + queueSize: queueSize, + unpermissionedFeeds: params.unpermissionedFeeds ?? false, + unpermissionedVrf: params.unpermissionedVrf ?? false, + }) + .accounts({ + oracleQueue: queueKeypair.publicKey, + authority: authorityKeypair.publicKey, + buffer: queueBuffer.publicKey, + systemProgram: SystemProgram.programId, + payer: payerKeypair.publicKey, + mint: tokenMint.publicKey, + }) + .instruction(), + anchor.web3.SystemProgram.createAccount({ + fromPubkey: payerKeypair.publicKey, + newAccountPubkey: crankBuffer.publicKey, + space: crankSize, + lamports: + await this.program.provider.connection.getMinimumBalanceForRentExemption( + crankSize + ), + programId: this.program.programId, + }), + await this.program.methods + .crankInit({ + name: Buffer.from("Crank").slice(0, 32), + metadata: Buffer.from("").slice(0, 64), + crankSize: params.crankSize, + }) + .accounts({ + crank: crankKeypair.publicKey, + queue: queueKeypair.publicKey, + buffer: crankBuffer.publicKey, + systemProgram: SystemProgram.programId, + payer: payerKeypair.publicKey, + }) + .instruction() + ); + signers.push(queueKeypair, queueBuffer, crankKeypair, crankBuffer); + + const finalTransactions: ( + | TransactionInstruction + | TransactionInstruction[] + )[] = []; + + const oracleAccounts = await Promise.all( + Array.from(Array(params.numOracles).keys()).map(async (n) => { + const name = `Oracle-${n + 1}`; + const tokenWalletKeypair = anchor.web3.Keypair.generate(); + const [oracleAccount, oracleBump] = OracleAccount.fromSeed( + this.program, + queueAccount, + tokenWalletKeypair.publicKey + ); + + this.logger.debug(chalkString(name, oracleAccount.publicKey)); + + const [permissionAccount, permissionBump] = PermissionAccount.fromSeed( + this.program, + authorityKeypair.publicKey, + queueAccount.publicKey, + oracleAccount.publicKey + ); + this.logger.debug( + chalkString(`Permission-${n + 1}`, permissionAccount.publicKey) + ); + + finalTransactions.push([ + SystemProgram.createAccount({ + fromPubkey: payerKeypair.publicKey, + newAccountPubkey: tokenWalletKeypair.publicKey, + lamports: + await this.program.provider.connection.getMinimumBalanceForRentExemption( + spl.AccountLayout.span + ), + space: spl.AccountLayout.span, + programId: spl.TOKEN_PROGRAM_ID, + }), + spl.Token.createInitAccountInstruction( + spl.TOKEN_PROGRAM_ID, + tokenMint.publicKey, + tokenWalletKeypair.publicKey, + programStateAccount.publicKey + ), + await this.program.methods + .oracleInit({ + name: Buffer.from(name).slice(0, 32), + metadata: Buffer.from("").slice(0, 128), + stateBump, + oracleBump, + }) + .accounts({ + oracle: oracleAccount.publicKey, + oracleAuthority: authorityKeypair.publicKey, + queue: queueKeypair.publicKey, + wallet: tokenWalletKeypair.publicKey, + programState: programStateAccount.publicKey, + systemProgram: SystemProgram.programId, + payer: payerKeypair.publicKey, + }) + .instruction(), + await this.program.methods + .permissionInit({}) + .accounts({ + permission: permissionAccount.publicKey, + authority: authorityKeypair.publicKey, + granter: queueAccount.publicKey, + grantee: oracleAccount.publicKey, + payer: payerKeypair.publicKey, + systemProgram: SystemProgram.programId, + }) + .instruction(), + await this.program.methods + .permissionSet({ + permission: { permitOracleHeartbeat: null }, + enable: true, + }) + .accounts({ + permission: permissionAccount.publicKey, + authority: authorityKeypair.publicKey, + }) + .instruction(), + ]); + signers.push(tokenWalletKeypair); + return { + oracleAccount, + name, + permissionAccount, + tokenWalletKeypair, + }; + }) + ); + + const createAccountSignatures = packAndSend( + this.program, + [ixns, finalTransactions], + signers, + payerKeypair.publicKey + ); + + const result = await program.provider.connection.confirmTransaction( + createAccountSignatures[-1] + ); + + return { + queueAccount, + crankPubkey: crankAccount.publicKey, + oracles: oracleAccounts.map((o) => o.oracleAccount.publicKey) ?? [], + }; +} diff --git a/libraries/sbv2-utils/src/solana.ts b/libraries/sbv2-utils/src/solana.ts deleted file mode 100644 index b035226..0000000 --- a/libraries/sbv2-utils/src/solana.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { Connection, SignatureResult } from "@solana/web3.js"; - -/** Watch a transaction and resolve when it is finalized */ -async function watchTransaction( - txn: string, - connection: Connection -): Promise { - console.log(`https://explorer.solana.com/tx/${txn}?cluster=devnet`); - connection.onSignature(txn, async (signatureResult: SignatureResult) => { - const response = await connection.getTransaction(txn); - console.log(JSON.stringify(response?.meta?.logMessages, undefined, 2)); - }); -} diff --git a/libraries/sbv2-utils/src/transaction.ts b/libraries/sbv2-utils/src/transaction.ts new file mode 100644 index 0000000..c972a16 --- /dev/null +++ b/libraries/sbv2-utils/src/transaction.ts @@ -0,0 +1,95 @@ +import * as anchor from "@project-serum/anchor"; +import { + ConfirmOptions, + Connection, + Keypair, + PublicKey, + TransactionInstruction, + TransactionSignature, +} from "@solana/web3.js"; +import { + packInstructions, + signTransactions, +} from "@switchboard-xyz/switchboard-v2"; + +export async function packAndSend( + program: anchor.Program, + ixnsBatches: (TransactionInstruction | TransactionInstruction[])[][], + signers: Keypair[], + feePayer: PublicKey +): Promise { + const signatures: Promise[] = []; + + for await (const batch of ixnsBatches) { + const { blockhash } = + await program.provider.connection.getLatestBlockhash(); + + const packedTransactions = packInstructions(batch, feePayer, blockhash); + const signedTransactions = signTransactions(packedTransactions, signers); + const signedTxs = await ( + program.provider as anchor.AnchorProvider + ).wallet.signAllTransactions(signedTransactions); + + for (let k = 0; k < packedTransactions.length; k += 1) { + const tx = signedTxs[k]; + const rawTx = tx.serialize(); + // signatures.push( + // program.provider.connection.sendRawTransaction(rawTx, { + // skipPreflight: true, + // maxRetries: 10, + // }) + // ); + signatures.push( + sendAndConfirmRawTransaction(program.provider.connection, rawTx, { + skipPreflight: true, + maxRetries: 10, + commitment: "confirmed", + }) + ); + // signatures.push( + // program.provider.connection.sendTransaction(tx, signers, { + // skipPreflight: true, + // maxRetries: 10, + // }) + // ); + } + + await Promise.all(signatures); + } + + return Promise.all(signatures); +} + +/** + * Send and confirm a raw transaction + * + * If `commitment` option is not specified, defaults to 'max' commitment. + */ +export async function sendAndConfirmRawTransaction( + connection: Connection, + rawTransaction: Buffer, + options: ConfirmOptions +): Promise { + const sendOptions = options && { + skipPreflight: options.skipPreflight, + preflightCommitment: options.preflightCommitment || options.commitment, + }; + const signature: TransactionSignature = await connection.sendRawTransaction( + rawTransaction, + sendOptions + ); + const status = ( + await connection.confirmTransaction( + signature as any, + options.commitment || "max" + ) + ).value; + + if (status.err) { + throw new Error( + `Raw transaction ${signature} failed (${JSON.stringify(status)})` + ); + } + + return signature; +} diff --git a/website/package.json b/website/package.json index fcd0fc7..b030854 100644 --- a/website/package.json +++ b/website/package.json @@ -11,7 +11,7 @@ "scripts": { "docusaurus": "docusaurus", "start": "docusaurus start", - "build": "For workspace website, run 'yarn docs:build' from the project root", + "build": "echo \"For workspace anchor-vrf-parser, run 'yarn docs:build' from the project root\" && exit 0", "build:site": "docusaurus build --out-dir public", "swizzle": "docusaurus swizzle", "deploy": "docusaurus build --out-dir public && docusaurus deploy --out-dir public", diff --git a/yarn.lock b/yarn.lock index d5c571b..c817f5a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3862,6 +3862,26 @@ superstruct "^0.14.2" tweetnacl "^1.0.0" +"@solana/web3.js@1.39.1": + version "1.39.1" + resolved "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.39.1.tgz#858ecd42ff2a5bcba3a4bb642a50194d77e2a578" + integrity sha512-Q7XnWTAiU7n7GcoINDAAMLO7CJHpm5kPK46HKwJi2x0cusHQ3WFa7QEp6aPzH7tuf7yl/Kw1lYitcwTVOvqARA== + dependencies: + "@babel/runtime" "^7.12.5" + "@ethersproject/sha2" "^5.5.0" + "@solana/buffer-layout" "^4.0.0" + bn.js "^5.0.0" + borsh "^0.7.0" + bs58 "^4.0.1" + buffer "6.0.1" + cross-fetch "^3.1.4" + jayson "^3.4.4" + js-sha3 "^0.8.0" + rpc-websockets "^7.4.2" + secp256k1 "^4.0.2" + superstruct "^0.14.2" + tweetnacl "^1.0.0" + "@solana/web3.js@^0.86.1": version "0.86.4" resolved "https://registry.npmjs.org/@solana/web3.js/-/web3.js-0.86.4.tgz" @@ -3927,28 +3947,6 @@ superstruct "^0.14.2" tweetnacl "^1.0.0" -"@solana/web3.js@^1.41.10": - version "1.41.10" - resolved "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.41.10.tgz#fb1bf7d8ca25f126a2166fed1733fe357298a076" - integrity sha512-2mPNoxGDt5jZ4MYA+aK7qKzvdXdN0niy7suYfkbrcgAWahJ/WSfPD2W0IvySDdLLfCQojSs7sdHIW+xsKV9dyQ== - dependencies: - "@babel/runtime" "^7.12.5" - "@ethersproject/sha2" "^5.5.0" - "@solana/buffer-layout" "^4.0.0" - "@solana/buffer-layout-utils" "^0.2.0" - bn.js "^5.0.0" - borsh "^0.7.0" - bs58 "^4.0.1" - buffer "6.0.1" - cross-fetch "^3.1.4" - fast-stable-stringify "^1.0.0" - jayson "^3.4.4" - js-sha3 "^0.8.0" - rpc-websockets "^7.4.2" - secp256k1 "^4.0.2" - superstruct "^0.14.2" - tweetnacl "^1.0.0" - "@solana/web3.js@^1.42.0": version "1.42.0" resolved "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.42.0.tgz#296e4bbab1fbfc198b3e9c3d94016c3876eb6a2c"