From 887b0e4b72282975fabf8fd85292a7e6dcfe868f Mon Sep 17 00:00:00 2001 From: Jack May Date: Fri, 13 Nov 2020 10:18:04 -0800 Subject: [PATCH] Add doc content and feedback (#13563) --- .gitignore | 2 + docs/sidebars.js | 50 ++-- docs/src/developing/builtin-programs.md | 63 ---- docs/src/developing/builtins/programs.md | 118 ++++++++ .../sysvars.md | 0 .../developing/deployed-programs/debugging.md | 13 +- .../deployed-programs/developing-c.md | 14 +- .../deployed-programs/developing-rust.md | 103 ++++--- .../developing/deployed-programs/examples.md | 12 +- docs/src/developing/deployed-programs/faq.md | 8 +- .../developing/programming-model/accounts.md | 79 ++--- .../calling-between-programs.md | 281 ++++++++++++++++++ .../programming-model/compute-budget.md | 61 ---- docs/src/developing/programming-model/cpi.md | 155 ---------- .../developing/programming-model/overview.md | 12 +- .../program-derived-addresses.md | 159 ---------- .../programming-model/runtime-features.md | 26 -- .../developing/programming-model/runtime.md | 127 ++++++++ .../programming-model/secpk1-instructions.md | 92 ------ .../programming-model/transactions.md | 97 +++++- docs/src/integrations/exchange.md | 7 +- docs/src/offline-signing/durable-nonce.md | 2 +- docs/src/transaction.md | 77 ----- 23 files changed, 754 insertions(+), 804 deletions(-) delete mode 100644 docs/src/developing/builtin-programs.md create mode 100644 docs/src/developing/builtins/programs.md rename docs/src/developing/{programming-model => builtins}/sysvars.md (100%) create mode 100644 docs/src/developing/programming-model/calling-between-programs.md delete mode 100644 docs/src/developing/programming-model/compute-budget.md delete mode 100644 docs/src/developing/programming-model/cpi.md delete mode 100644 docs/src/developing/programming-model/program-derived-addresses.md delete mode 100644 docs/src/developing/programming-model/runtime-features.md create mode 100644 docs/src/developing/programming-model/runtime.md delete mode 100644 docs/src/developing/programming-model/secpk1-instructions.md delete mode 100644 docs/src/transaction.md diff --git a/.gitignore b/.gitignore index de94a88ef..5492e8e36 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ /config/ +.cache + # log files *.log log-*.txt diff --git a/docs/sidebars.js b/docs/sidebars.js index c4a5918b6..205aebefa 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -61,32 +61,13 @@ module.exports = { "Developing": [ { type: "category", - label: "Programming model", + label: "Programming Model", items: [ "developing/programming-model/overview", "developing/programming-model/transactions", "developing/programming-model/accounts", - "developing/programming-model/sysvars", - "developing/programming-model/runtime-features", - "developing/programming-model/compute-budget", - "developing/programming-model/cpi", - "developing/programming-model/program-derived-addresses", - "developing/programming-model/secpk1-instructions", - - ], - }, - "developing/builtin-programs", - { - type: "category", - label: "Deployed programs", - items: [ - "developing/deployed-programs/overview", - "developing/deployed-programs/developing-rust", - "developing/deployed-programs/developing-c", - "developing/deployed-programs/deploying", - "developing/deployed-programs/debugging", - "developing/deployed-programs/examples", - "developing/deployed-programs/faq", + "developing/programming-model/runtime", + "developing/programming-model/calling-between-programs", ], }, { @@ -97,6 +78,27 @@ module.exports = { "developing/clients/javascript-api", ], }, + { + type: "category", + label: "Builtins", + items: [ + "developing/builtins/programs", + "developing/builtins/sysvars", + ], + }, + { + type: "category", + label: "Deployed Programs", + items: [ + "developing/deployed-programs/overview", + "developing/deployed-programs/developing-rust", + "developing/deployed-programs/developing-c", + "developing/deployed-programs/deploying", + "developing/deployed-programs/debugging", + "developing/deployed-programs/examples", + "developing/deployed-programs/faq", + ], + }, "developing/backwards-compatibility", ], "Integrating": ["integrations/exchange"], @@ -199,10 +201,6 @@ module.exports = { "implemented-proposals/ed_overview/ed_references", ], }, - "implemented-proposals/ed_overview/ed_storage_rent_economics", - "implemented-proposals/ed_overview/ed_economic_sustainability", - "implemented-proposals/ed_overview/ed_mvp", - "implemented-proposals/ed_overview/ed_references", ], }, "implemented-proposals/abi-management", diff --git a/docs/src/developing/builtin-programs.md b/docs/src/developing/builtin-programs.md deleted file mode 100644 index 0e23bf818..000000000 --- a/docs/src/developing/builtin-programs.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: "Builtin programs" ---- - -Solana contains a small handful of builtin programs, which are required to run -validator nodes. Unlike third-party programs, the builtin programs are part of -the validator implementation and can be upgraded as part of cluster upgrades. -Upgrades may occur to add features, fix bugs, or improve performance. Interface -changes to individual instructions should rarely, if ever, occur. Instead, when -change is needed, new instructions are added and previous ones are marked -deprecated. Apps can upgrade on their own timeline without concern of breakages -across upgrades. - -The builtin programs include the System, Config, Stake, Vote, and BPFLoader -programs. For each, we provide the program ID and describe each supported -instruction. A transaction can mix and match instructions from different -programs, as well include instructions from third-party programs. - -## System Program - -Create accounts and transfer lamports between them - -- Program ID: `11111111111111111111111111111111` -- Instructions: [SystemInstruction](https://docs.rs/solana-sdk/VERSION_FOR_DOCS_RS/solana_sdk/system_instruction/enum.SystemInstruction.html) - -## Config Program - -Add configuration data to the chain and the list of public keys that are permitted to modify it - -- Program ID: `Config1111111111111111111111111111111111111` -- Instructions: [config_instruction](https://docs.rs/solana-config-program/VERSION_FOR_DOCS_RS/solana_config_program/config_instruction/index.html) - -Unlike the other programs, the Config program does not define any individual -instructions. It has just one implicit instruction, a "store" instruction. Its -instruction data is a set of keys that gate access to the account, and the -data to store in it. - -## Stake Program - -Create stake accounts and delegate it to validators - -- Program ID: `Stake11111111111111111111111111111111111111` -- Instructions: [StakeInstruction](https://docs.rs/solana-stake-program/VERSION_FOR_DOCS_RS/solana_stake_program/stake_instruction/enum.StakeInstruction.html) - -## Vote Program - -Create vote accounts and vote on blocks - -- Program ID: `Vote111111111111111111111111111111111111111` -- Instructions: [VoteInstruction](https://docs.rs/solana-vote-program/VERSION_FOR_DOCS_RS/solana_vote_program/vote_instruction/enum.VoteInstruction.html) - -## BPF Loader - -Add programs to the chain and execute them. - -- Program ID: `BPFLoader1111111111111111111111111111111111` -- Instructions: [LoaderInstruction](https://docs.rs/solana-sdk/VERSION_FOR_DOCS_RS/solana_sdk/loader_instruction/enum.LoaderInstruction.html) - -The BPF Loader marks itself as its "owner" of the executable account it -creates to store your program. When a user invokes an instruction via a -program ID, the Solana runtime will load both your executable account and its -owner, the BPF Loader. The runtime then passes your program to the BPF Loader -to process the instruction. diff --git a/docs/src/developing/builtins/programs.md b/docs/src/developing/builtins/programs.md new file mode 100644 index 000000000..86c5a6537 --- /dev/null +++ b/docs/src/developing/builtins/programs.md @@ -0,0 +1,118 @@ +--- +title: "Builtin Programs" +--- + +Solana contains a small handful of builtin programs, which are required to run +validator nodes. Unlike third-party programs, the builtin programs are part of +the validator implementation and can be upgraded as part of cluster upgrades. +Upgrades may occur to add features, fix bugs, or improve performance. Interface +changes to individual instructions should rarely, if ever, occur. Instead, when +change is needed, new instructions are added and previous ones are marked +deprecated. Apps can upgrade on their own timeline without concern of breakages +across upgrades. + +For each builtin program the program id and description each supported +instruction is provided. A transaction can mix and match instructions from different +programs, as well include instructions from deployed programs. + +## System Program + +Create accounts and transfer lamports between them + +- Program id: `11111111111111111111111111111111` +- Instructions: [SystemInstruction](https://docs.rs/solana-sdk/VERSION_FOR_DOCS_RS/solana_sdk/system_instruction/enum.SystemInstruction.html) + +## Config Program + +Add configuration data to the chain and the list of public keys that are permitted to modify it + +- Program id: `Config1111111111111111111111111111111111111` +- Instructions: [config_instruction](https://docs.rs/solana-config-program/VERSION_FOR_DOCS_RS/solana_config_program/config_instruction/index.html) + +Unlike the other programs, the Config program does not define any individual +instructions. It has just one implicit instruction, a "store" instruction. Its +instruction data is a set of keys that gate access to the account, and the +data to store in it. + +## Stake Program + +Create stake accounts and delegate it to validators + +- Program id: `Stake11111111111111111111111111111111111111` +- Instructions: [StakeInstruction](https://docs.rs/solana-stake-program/VERSION_FOR_DOCS_RS/solana_stake_program/stake_instruction/enum.StakeInstruction.html) + +## Vote Program + +Create vote accounts and vote on blocks + +- Program id: `Vote111111111111111111111111111111111111111` +- Instructions: [VoteInstruction](https://docs.rs/solana-vote-program/VERSION_FOR_DOCS_RS/solana_vote_program/vote_instruction/enum.VoteInstruction.html) + +## BPF Loader + +Add programs to the chain and execute them. + +- Program id: `BPFLoader1111111111111111111111111111111111` +- Instructions: [LoaderInstruction](https://docs.rs/solana-sdk/VERSION_FOR_DOCS_RS/solana_sdk/loader_instruction/enum.LoaderInstruction.html) + +The BPF Loader marks itself as its "owner" of the executable account it +creates to store your program. When a user invokes an instruction via a +program id, the Solana runtime will load both your executable account and its +owner, the BPF Loader. The runtime then passes your program to the BPF Loader +to process the instruction. + +## Secp256k1 Program + +Verify secp256k1 public key recovery operations (ecrecover). + +- Program id: `KeccakSecp256k11111111111111111111111111111` +- Instructions: [new_secp256k1_instruction](https://github.com/solana-labs/solana/blob/c1f3f9d27b5f9534f9a37704bae1d690d4335b6b/programs/secp256k1/src/lib.rs#L18) + +The secp256k1 program processes an instruction which takes in as the first byte +a count of the following struct serialized in the instruction data: + +``` +struct Secp256k1SignatureOffsets { + secp_signature_key_offset: u16, // offset to [signature,recovery_id,etherum_address] of 64+1+20 bytes + secp_signature_instruction_index: u8, // instruction index to find data + secp_pubkey_offset: u16, // offset to [signature,recovery_id] of 64+1 bytes + secp_signature_instruction_index: u8, // instruction index to find data + secp_message_data_offset: u16, // offset to start of message data + secp_message_data_size: u16, // size of message data + secp_message_instruction_index: u8, // index of instruction data to get message data +} +``` + +Pseudo code of the operation: +``` +process_instruction() { + for i in 0..count { + // i'th index values referenced: + instructions = &transaction.message().instructions + signature = instructions[secp_signature_instruction_index].data[secp_signature_offset..secp_signature_offset + 64] + recovery_id = instructions[secp_signature_instruction_index].data[secp_signature_offset + 64] + ref_eth_pubkey = instructions[secp_pubkey_instruction_index].data[secp_pubkey_offset..secp_pubkey_offset + 32] + message_hash = keccak256(instructions[secp_message_instruction_index].data[secp_message_data_offset..secp_message_data_offset + secp_message_data_size]) + pubkey = ecrecover(signature, recovery_id, message_hash) + eth_pubkey = keccak256(pubkey[1..])[12..] + if eth_pubkey != ref_eth_pubkey { + return Error + } + } + return Success +} +``` + +This allows the user to specify any instruction data in the transaction for +signature and message data. By specifying a special instructions sysvar, one can +also receive data from the transaction itself. + +Cost of the transaction will count the number of signatures to verify multiplied +by the signature cost verify multiplier. + +### Optimization notes + +The operation will have to take place after (at least partial) deserialization, +but all inputs come from the transaction data itself, this allows it to be +relatively easy to execute in parallel to transaction processing and PoH +verification. diff --git a/docs/src/developing/programming-model/sysvars.md b/docs/src/developing/builtins/sysvars.md similarity index 100% rename from docs/src/developing/programming-model/sysvars.md rename to docs/src/developing/builtins/sysvars.md diff --git a/docs/src/developing/deployed-programs/debugging.md b/docs/src/developing/deployed-programs/debugging.md index b41481089..9b0c93205 100644 --- a/docs/src/developing/deployed-programs/debugging.md +++ b/docs/src/developing/deployed-programs/debugging.md @@ -9,15 +9,8 @@ that will allow RPC clients to interact with their program. ## Running unit tests -TODO - -## Running on a Local Cluster - -TODO - -## Transaction Simulation - -TODO +- [Testing with Rust](developing-rust.md#how-to-test) +- [Testing with C](developing-c.md#how-to-test) ## Logging @@ -92,7 +85,7 @@ operations they wish to profile. program](developing-c.md#compute-budget) See [compute -budget](developing/programming-model/../../../programming-model/compute-budget.md) +budget](developing/programming-model/../../../programming-model/runtime.md/#compute-budget) for more information. ## ELF Dump diff --git a/docs/src/developing/deployed-programs/developing-c.md b/docs/src/developing/deployed-programs/developing-c.md index 4c631be81..fd19dd6c3 100644 --- a/docs/src/developing/deployed-programs/developing-c.md +++ b/docs/src/developing/deployed-programs/developing-c.md @@ -178,7 +178,7 @@ to log a message containing the remaining number of compute units the program may consume before execution is halted See [compute -budget](developing/programming-model/../../../programming-model/compute-budget.md) +budget](developing/programming-model/../../../programming-model/runtime.md/#compute-budget) for more information. ## ELF Dump @@ -200,14 +200,4 @@ $ make dump_ ## Examples -TODO - -### Logging - -### Transferring Lamports - -### Writing Account Data - -### Custom Heap - -### Cross-program Invocations +The [Solana Program Library github](https://github.com/solana-labs/solana-program-library/tree/master/examples/c) repo contains a collection of C examples diff --git a/docs/src/developing/deployed-programs/developing-rust.md b/docs/src/developing/deployed-programs/developing-rust.md index 5ece9dde0..cd5c2b070 100644 --- a/docs/src/developing/deployed-programs/developing-rust.md +++ b/docs/src/developing/deployed-programs/developing-rust.md @@ -28,10 +28,11 @@ features = [] Solana Rust programs may depend directly on each other in order to gain access to instruction helpers when making [cross-program -invocations](developing/../../programming-model/cpi.md). When doing so it's -important to not pull in the dependent program's entrypoint symbols because they -may conflict with the program's own. To avoid this ,programs should define an -`exclude_entrypoint` feature in `Cargo.toml`j and use to exclude the entrypoint. +invocations](developing/../../programming-model/calling-between-programs.md#cross-program-invocations). +When doing so it's important to not pull in the dependent program's entrypoint +symbols because they may conflict with the program's own. To avoid this +,programs should define an `exclude_entrypoint` feature in `Cargo.toml`j and use +to exclude the entrypoint. - [Define the feature](https://github.com/solana-labs/solana-program-library/blob/a5babd6cbea0d3f29d8c57d2ecbbd2a2bd59c8a9/token/program/Cargo.toml#L12) @@ -48,29 +49,19 @@ using the `exclude_entrypoint` feature. At a minimum, Solana Rust programs must pull in the [solana-program](https://crates.io/crates/solana-program) crate. -Programs are constrained to run deterministically, so random numbers are not -available. Sometimes a program may depend on a crate that depends itself on -`rand` even if the program does not use any of the random number functionality. -If a program depends on `rand`, the compilation will fail because there is no -`get-random` support for Solana. The error will typically look like this: +Solana BPF programs have some [restrictions](#Restrictions) that may prevent the +inclusion of some crates as dependencies or require special handling. -``` -error: target is not supported, for more information see: https://docs.rs/getrandom/#unsupported-targets - --> /Users/jack/.cargo/registry/src/github.com-1ecc6299db9ec823/getrandom-0.1.14/src/lib.rs:257:9 - | -257 | / compile_error!("\ -258 | | target is not supported, for more information see: \ -259 | | https://docs.rs/getrandom/#unsupported-targets\ -260 | | "); - | |___________^ -``` - -To work around this dependency issue, add the following dependency to the -program's `Cargo.toml`: - -``` -getrandom = { version = "0.1.14", features = ["dummy"] } -``` +For example: +- Crates that require the architecture be a subset of the ones supported by the + official toolchain. There is no workaround for this unless that crate is + forked and BPF added to that those architecture checks. +- Crates may depend on `rand` which is not supported in Solana's deterministic + program environment. To include a `rand` dependent crate refer to [Depending + on Rand](#depending-on-rand). +- Crates may overflow the stack even if the stack overflowing code isn't + included in the program itself. For more information refer to + [Stack](overview.md#stack). ## How to Build @@ -96,7 +87,19 @@ $ cargo build-bpf ## How to Test -TODO +Solana programs can be unit tested via the traditional `cargo test` mechanism by +exercising program functions directly. + +To help facilitate testing in an environment that more closely matches a live +cluster, developers can use the +[`program-test`](https://crates.io/crates/solana-program-test) crate. The +`program-test` crate starts up a local instance of the runtime and allows tests +to send multiple transactions while keeping state for the duration of the test. + +For more information the [test in sysvar +example](https://github.com/solana-labs/solana-program-library/blob/master/examples/rust/sysvar/tests/functional.rs) +shows how an instruction containing syavar account is sent and processed by the +program. ## Program Entrypoint @@ -240,9 +243,35 @@ single-threaded environment, and must be deterministic: should be used instead. - The runtime enforces a limit on the number of instructions a program can execute during the processing of one instruction. See [computation - budget](developing/programming-model/compute-budget.md) for more + budget](developing/programming-model/runtime.md#compute-budget) for more information. +## Depending on Rand + +Programs are constrained to run deterministically, so random numbers are not +available. Sometimes a program may depend on a crate that depends itself on +`rand` even if the program does not use any of the random number functionality. +If a program depends on `rand`, the compilation will fail because there is no +`get-random` support for Solana. The error will typically look like this: + +``` +error: target is not supported, for more information see: https://docs.rs/getrandom/#unsupported-targets + --> /Users/jack/.cargo/registry/src/github.com-1ecc6299db9ec823/getrandom-0.1.14/src/lib.rs:257:9 + | +257 | / compile_error!("\ +258 | | target is not supported, for more information see: \ +259 | | https://docs.rs/getrandom/#unsupported-targets\ +260 | | "); + | |___________^ +``` + +To work around this dependency issue, add the following dependency to the +program's `Cargo.toml`: + +``` +getrandom = { version = "0.1.14", features = ["dummy"] } +``` + ## Logging Rust's `println!` macro is computationally expensive and not supported. Instead @@ -337,7 +366,7 @@ to log a message containing the remaining number of compute units the program may consume before execution is halted See [compute -budget](developing/programming-model/../../../programming-model/compute-budget.md) +budget](developing/programming-model/../../../programming-model/runtime.md#compute-budget) for more information. ## ELF Dump @@ -359,16 +388,6 @@ $ cargo build-bpf --dump ## Examples -TODO - -### Logging - -### Transferring Lamports - -### Writing Account Data - -### Using a Sysvar - -### Custom Heap - -### Cross-program Invocations +The [Solana Program Library +github](https://github.com/solana-labs/solana-program-library/tree/master/examples/rust) +repo contains a collection of Rust examples. diff --git a/docs/src/developing/deployed-programs/examples.md b/docs/src/developing/deployed-programs/examples.md index 2227d277c..7ab9e8ca7 100644 --- a/docs/src/developing/deployed-programs/examples.md +++ b/docs/src/developing/deployed-programs/examples.md @@ -2,6 +2,7 @@ title: "Examples" --- + ## Helloworld Hello World is a project that demonstrates how to use the Solana Javascript API @@ -13,7 +14,7 @@ The project comprises of: - A client that can send a "hello" to an account and get back the number of times "hello" has been sent -## Build and run Hello World program +### Build and Run First fetch the latest version of the example code: @@ -30,7 +31,7 @@ Next, follow the steps in the git repository's [Break](https://break.solana.com/) is a React app that gives users a visceral feeling for just how fast and high-performance the Solana network really is. Can -you _break_ the Solana blockchain? During a 15 second playthough, each click of +you _break_ the Solana blockchain? During a 15 second play-though, each click of a button or keystroke sends a new transaction to the cluster. Smash the keyboard as fast as you can and watch your transactions get finalized in real time while the network takes it all in stride! @@ -43,7 +44,7 @@ transfer the tokens. [Click here to play Break](https://break.solana.com/) -## Build and run Break locally +### Build and Run First fetch the latest version of the example code: @@ -54,3 +55,8 @@ $ cd break Next, follow the steps in the git repository's [README](https://github.com/solana-labs/break/blob/master/README.md). + +## Language Specific + +- [Rust](developing-rust.md#examples) +- [C](developing-c.md#examples) diff --git a/docs/src/developing/deployed-programs/faq.md b/docs/src/developing/deployed-programs/faq.md index f34f7f727..9aa853a14 100644 --- a/docs/src/developing/deployed-programs/faq.md +++ b/docs/src/developing/deployed-programs/faq.md @@ -15,7 +15,7 @@ This error means that that cross-program invocation exceeded the allowed invocation call depth. See [cross-program invocation Call -Depth](developing/programming-model/cpi.md#call-depth) +Depth](developing/programming-model/calling-between-programs.md#call-depth) ## `CallDepthExceeded` error @@ -26,7 +26,7 @@ See [call depth](overview.md#call-depth) ## Computational constraints See [computational -constraints](developing/programming-model/compute-budget.md) +constraints](developing/programming-model/runtime.md#compute-budget) ## Float Rust types @@ -64,9 +64,9 @@ an account is expected to be signed but is not. An implementation of a program might also cause this error when performing a cross-program invocation that requires a signed program address, but the passed -signer seeds passed to [`invoke_signed`](developing/programming-model/cpi.md) +signer seeds passed to [`invoke_signed`](developing/programming-model/calling-between-programs.md) don't match the signer seeds used to create the program address -[`create_program_address`](developing/programming-model/program-derived-addresses.md). +[`create_program_address`](developing/programming-model/calling-between-programs.md#program-derived-addresses). ## `rand` Rust dependency causes compilation failure diff --git a/docs/src/developing/programming-model/accounts.md b/docs/src/developing/programming-model/accounts.md index 1e8bf710d..6abc7ce04 100644 --- a/docs/src/developing/programming-model/accounts.md +++ b/docs/src/developing/programming-model/accounts.md @@ -13,12 +13,13 @@ tells the runtime who is allowed to access the data and how. Unlike a file, the account includes metadata for the lifetime of the file. That lifetime is expressed in "tokens", which is a number of fractional native tokens, called _lamports_. Accounts are held in validator memory and pay -["rent"](#rent) to stay there. Each validator periodically scans all -accounts and collects rent. Any account that drops to zero lamports is purged. +["rent"](#rent) to stay there. Each validator periodically scans all accounts +and collects rent. Any account that drops to zero lamports is purged. Accounts +can also be marked [rent-exempt](#rent-exemption) if they contain a sufficnet +number of lamports. In the same way that a Linux user uses a path to look up a file, a Solana client -uses an _address_ to look up an account. The address is usually a 256-bit public -key. +uses an _address_ to look up an account. The address is a 256-bit public key. ## Signers @@ -32,19 +33,22 @@ then use that information to make authority decisions. ## Read-only -Transactions can mark some accounts as _read-only accounts_. The runtime permits -read-only accounts to be read concurrently by multiple programs. If a program -attempts to modify a read-only account, the transaction is rejected by the -runtime. +Transactions can [indicate](transactions.md#message-header-format) that some of +the accounts it references be treated as _read-only accounts_ in order to enable +parallel account processing between transactions. The runtime permits read-only +accounts to be read concurrently by multiple programs. If a program attempts to +modify a read-only account, the transaction is rejected by the runtime. ## Executable -If an account is marked "executable" in its metadata, it can be used by a -_loader_ to run programs. For example, a BPF-compiled program is marked -executable by the BPF loader during deployment once the loader has determined -that the BPF bytecode in the account's data is valid. No program is allowed to -modify the contents of an executable account once deployed and executable mark -is permanent. +If an account is marked "executable" in its metadata then it is considered a +program which can be executed by including the account's public key an +instruction's [program id](transactions.md#program-id). Accounts are marked as +executable during a successful program deployment process by the loader that +owns the account. For example, during BPF program deployment, once the loader +has determined that the BPF bytecode in the account's data is valid, the loader +permanently marks the program account as executable. Once executable, the +runtime enforces that the account's data (the program) is immutable. ## Creating @@ -56,7 +60,7 @@ megabytes. An account address can be any arbitrary 256 bit value, and there are mechanisms for advanced users to create derived addresses (`SystemProgram::CreateAccountWithSeed`, -[`Pubkey::CreateProgramAddress`](program-derived-addresses.md)). +[`Pubkey::CreateProgramAddress`](calling-between-programs.md#program-derived-addresses)). Accounts that have never been created via the system program can also be passed to programs. When an instruction references an account that hasn't been @@ -73,52 +77,13 @@ operation the program controls or performs. A created account is initialized to be _owned_ by a built-in program called the System program and is called a _system account_ aptly. An account includes -"owner" metadata. The owner is a program ID. The runtime grants the program -write access to the account if its ID matches the owner. For the case of the +"owner" metadata. The owner is a program id. The runtime grants the program +write access to the account if its id matches the owner. For the case of the System program, the runtime allows clients to transfer lamports and importantly -_assign_ account ownership, meaning changing owner to different program ID. If +_assign_ account ownership, meaning changing owner to different program id. If an account is not owned by a program, the program is only permitted to read its data and credit the account. -## Runtime Capability of Programs - -The runtime only permits the owner program to debit the account or modify its -data. The program then defines additional rules for whether the client can -modify accounts it owns. In the case of the System program, it allows users to -transfer lamports by recognizing transaction signatures. If it sees the client -signed the transaction using the keypair's _private key_, it knows the client -authorized the token transfer. - -In other words, the entire set of accounts owned by a given program can be -regarded as a key-value store where a key is the account address and value is -program-specific arbitrary binary data. A program author can decide how to -manage the program's whole state as possibly many accounts. - -After the runtime executes each of the transaction's instructions, it uses the -account metadata to verify that the access policy was not violated. If a program -violates the policy, the runtime discards all account changes made by all -instructions in the transaction and marks the transaction as failed. - -### Policy - -After a program has processed an instruction the runtime verifies that the -program only performed operations it was permitted to, and that the results -adhere to the runtime policy. - -The policy is as follows: -- Only the owner of the account may change owner. - - And only if the account is writable. - - And only if the data is zero-initialized or empty. -- An account not assigned to the program cannot have its balance decrease. -- The balance of read-only and executable accounts may not change. -- Only the system program can change the size of the data and only if the system - program owns the account. -- Only the owner may change account data. - - And if the account is writable. - - And if the account is not executable. -- Executable is one-way (false->true) and only the account owner may set it. -- No one modification to the rent_epoch associated with this account. - ## Rent Keeping accounts alive on Solana incurs a storage cost called _rent_ because the diff --git a/docs/src/developing/programming-model/calling-between-programs.md b/docs/src/developing/programming-model/calling-between-programs.md new file mode 100644 index 000000000..fdb392ac4 --- /dev/null +++ b/docs/src/developing/programming-model/calling-between-programs.md @@ -0,0 +1,281 @@ +--- +title: Calling Between Programs +--- + +## Cross-Program Invocations + +The Solana runtime allows programs to call each other via a mechanism called +cross-program invocation. Calling between programs is achieved by one program +invoking an instruction of the other. The invoking program is halted until the +invoked program finishes processing the instruction. + +For example, a client could create a transaction that modifies two accounts, +each owned by separate on-chain programs: + +```rust,ignore +let message = Message::new(vec![ + token_instruction::pay(&alice_pubkey), + acme_instruction::launch_missiles(&bob_pubkey), +]); +client.send_and_confirm_message(&[&alice_keypair, &bob_keypair], &message); +``` + +A client may to instead allow the `acme` program to conveniently invoke `token` +instructions on the client's behalf: + +```rust,ignore +let message = Message::new(vec![ + acme_instruction::pay_and_launch_missiles(&alice_pubkey, &bob_pubkey), +]); +client.send_and_confirm_message(&[&alice_keypair, &bob_keypair], &message); +``` + +Given two on-chain programs `token` and `acme`, each implementing instructions +`pay()` and `launch_missiles()` respectively, acme can be implemented with a +call to a function defined in the `token` module by issuing a cross-program +invocation: + +```rust,ignore +mod acme { + use token_instruction; + + fn launch_missiles(accounts: &[AccountInfo]) -> Result<()> { + ... + } + + fn pay_and_launch_missiles(accounts: &[AccountInfo]) -> Result<()> { + let alice_pubkey = accounts[1].key; + let instruction = token_instruction::pay(&alice_pubkey); + invoke(&instruction, accounts)?; + + launch_missiles(accounts)?; + } +``` + +`invoke()` is built into Solana's runtime and is responsible for routing the +given instruction to the `token` program via the instruction's `program_id` +field. + +Note that `invoke` requires the caller to pass all the accounts required by the +instruction being invoked. This means that both the executable account (the +ones that matches the instruction's program id) and the accounts passed to the +instruction procesor. + +Before invoking `pay()`, the runtime must ensure that `acme` didn't modify any +accounts owned by `token`. It does this by applying the runtime's policy to the +current state of the accounts at the time `acme` calls `invoke` vs. the initial +state of the accounts at the beginning of the `acme`'s instruction. After +`pay()` completes, the runtime must again ensure that `token` didn't modify any +accounts owned by `acme` by again applying the runtime's policy, but this time +with the `token` program ID. Lastly, after `pay_and_launch_missiles()` +completes, the runtime must apply the runtime policy one more time, where it +normally would, but using all updated `pre_*` variables. If executing +`pay_and_launch_missiles()` up to `pay()` made no invalid account changes, +`pay()` made no invalid changes, and executing from `pay()` until +`pay_and_launch_missiles()` returns made no invalid changes, then the runtime +can transitively assume `pay_and_launch_missiles()` as whole made no invalid +account changes, and therefore commit all these account modifications. + +### Instructions that require privileges + +The runtime uses the privileges granted to the caller program to determine what +privileges can be extended to the callee. Privileges in this context refer to +signers and writable accounts. For example, if the instruction the caller is +processing contains a signer or writable account, then the caller can invoke an +instruction that also contains that signer and/or writable account. + +This privilege extension relies on the fact that programs are immutable. In the +case of the `acme` program, the runtime can safely treat the transaction's +signature as a signature of a `token` instruction. When the runtime sees the +`token` instruction references `alice_pubkey`, it looks up the key in the `acme` +instruction to see if that key corresponds to a signed account. In this case, it +does and thereby authorizes the `token` program to modify Alice's account. + +### Program signed accounts + +Programs can issue instructions that contain signed accounts that were not +signed in the original transaction by using [Program derived +addresses](#program-derived-addresses). + +To sign an account with program derived addresses, a program may +`invoke_signed()`. + +```rust,ignore + invoke_signed( + &instruction, + accounts, + &[&["First addresses seed"], + &["Second addresses first seed", "Second addresses second seed"]], + )?; +``` + +### Call Depth + +Cross-program invocations allow programs to invoke other programs directly but +the depth is constrained currently to 4. + +### Reentrancy + +Reentrancy is currently limited to direct self recursion capped at a fixed +depth. This restriction prevents situations where a program might invoke another +from an intermediary state without the knowledge that it might later be called +back into. Direct recursion gives the program full control of its state at the +point that it gets called back. + +## Program Derived Addresses + +Program derived addresses allow programmaticly generated signature to be used +when [calling between programs](#cross-program-invocations). + +Using a program derived address, a program may be given the authority over an +account and later transfer that authority to another. This is possible because +the program can act as the signer in the transaction that gives authority. + +For example, if two users want to make a wager on the outcome of a game in +Solana, they must each transfer their wager's assets to some intermediary that +will honor their agreement. Currently, there is no way to implement this +intermediary as a program in Solana because the intermediary program cannot +transfer the assets to the winner. + +This capability is necessary for many DeFi applications since they require +assets to be transferred to an escrow agent until some event occurs that +determines the new owner. + +- Decentralized Exchanges that transfer assets between matching bid and ask + orders. + +- Auctions that transfer assets to the winner. + +- Games or prediction markets that collect and redistribute prizes to the + winners. + +Program derived address: + +1. Allow programs to control specific addresses, called program addresses, in + such a way that no external user can generate valid transactions with + signatures for those addresses. + +2. Allow programs to programmatically sign for programa addresses that are + present in instructions invoked via [Cross-Program Invocations](#cross-program-invocations). + +Given the two conditions, users can securely transfer or assign the authority of +on-chain assets to program addresses and the program can then assign that +authority elsewhere at its discretion. + +### Private keys for program addresses + +A Program address does not lie on the ed25519 curve and therefore has no valid +private key associated with it, and thus generating a signature for it is +impossible. While it has no private key of its own, it can be used by a program +to issue an instruction that includes the Program address as a signer. + +### Hash-based generated program addresses + +Program addresses are deterministically derived from a collection of seeds and a +program id using a 256-bit pre-image resistant hash function. Program address +must not lie on the ed25519 curve to ensure there is no associated private key. +During generation an error will be returned if the address is found to lie on +the curve. There is about a 50/50 change of this happening for a given +collection of seeds and program id. If this occurs a different set of seeds or +a seed bump (additional 8 bit seed) can be used to find a valid program address +off the curve. + +Deterministic program addresses for programs follow a similar derivation path as +Accounts created with `SystemInstruction::CreateAccountWithSeed` which is +implemented with `system_instruction::create_address_with_seed`. + +For reference that implementation is as follows: + +```rust,ignore +pub fn create_address_with_seed( + base: &Pubkey, + seed: &str, + program_id: &Pubkey, +) -> Result { + if seed.len() > MAX_ADDRESS_SEED_LEN { + return Err(SystemError::MaxSeedLengthExceeded); + } + + Ok(Pubkey::new( + hashv(&[base.as_ref(), seed.as_ref(), program_id.as_ref()]).as_ref(), + )) +} +``` + +Programs can deterministically derive any number of addresses by using seeds. +These seeds can symbolically identify how the addresses are used. + +From `Pubkey`:: + +```rust,ignore +/// Generate a derived program address +/// * seeds, symbolic keywords used to derive the key +/// * program_id, program that the address is derived for +pub fn create_program_address( + seeds: &[&[u8]], + program_id: &Pubkey, +) -> Result +``` + +### Using program addresses + +Clients can use the `create_program_address` function to generate a destination +address. + +```rust,ignore +// deterministically derive the escrow key +let escrow_pubkey = create_program_address(&[&["escrow"]], &escrow_program_id); + +// construct a transfer message using that key +let message = Message::new(vec![ + token_instruction::transfer(&alice_pubkey, &escrow_pubkey, 1), +]); + +// process the message which transfer one 1 token to the escrow +client.send_and_confirm_message(&[&alice_keypair], &message); +``` + +Programs can use the same function to generate the same address. In the function +below the program issues a `token_instruction::transfer` from a program address +as if it had the private key to sign the transaction. + +```rust,ignore +fn transfer_one_token_from_escrow( + program_id: &Pubkey, + keyed_accounts: &[KeyedAccount] +) -> Result<()> { + + // User supplies the destination + let alice_pubkey = keyed_accounts[1].unsigned_key(); + + // Deterministically derive the escrow pubkey. + let escrow_pubkey = create_program_address(&[&["escrow"]], program_id); + + // Create the transfer instruction + let instruction = token_instruction::transfer(&escrow_pubkey, &alice_pubkey, 1); + + // The runtime deterministically derives the key from the currently + // executing program ID and the supplied keywords. + // If the derived address matches a key marked as signed in the instruction + // then that key is accepted as signed. + invoke_signed(&instruction, &[&["escrow"]])? +} +``` + +### Instructions that require signers + +The addresses generated with `create_program_address` are indistinguishable from +any other public key. The only way for the runtime to verify that the address +belongs to a program is for the program to supply the seeds used to generate the +address. + +The runtime will internally call `create_program_address`, and compare the +result against the addresses supplied in the instruction. + +## Examples + +Refer to [Developing with +Rust](developing/deployed-programs/../../../deployed-programs/developing-rust.md#examples) +and [Developing with +C](developing/deployed-programs/../../../deployed-programs/developing-c.md#examples) +for examples of how to use cross-program invocation. diff --git a/docs/src/developing/programming-model/compute-budget.md b/docs/src/developing/programming-model/compute-budget.md deleted file mode 100644 index 6d0445dd1..000000000 --- a/docs/src/developing/programming-model/compute-budget.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: "Compute budget" ---- - -To prevent a program from abusing computation resources each instruction in a -transaction is given a compute budget. The budget consists of computation units -that are consumed as the program performs various operations and bounds that the -program may not exceed. When the program consumes its entire budget or exceeds -a bound then the runtime halts the program and returns an error. - -The following operations incur a compute cost: -- Executing BPF instructions -- Calling system calls - - logging - - creating program addresses - - cross-program invocations - - ... - -For cross-program invocations the programs invoked inherit the budget of their -parent. If an invoked program consume the budget or exceeds a bound the entire -invocation chain and the parent are halted. - -The current [compute -budget](https://github.com/solana-labs/solana/blob/d3a3a7548c857f26ec2cb10e270da72d373020ec/sdk/src/process_instruction.rs#L65) -can be found in the Solana SDK. - -For example, if the current budget is: - -```rust -max_units: 200,000, -log_units: 100, -log_u64_units: 100, -create_program address units: 1500, -invoke_units: 1000, -max_invoke_depth: 4, -max_call_depth: 64, -stack_frame_size: 4096, -log_pubkey_units: 100, -``` - -Then the program -- Could execute 200,000 BPF instructions if it does nothing else -- Could log 2,000 log messages -- Can not exceed 4k of stack usage -- Can not exceed a BPF call depth of 64 -- Cannot exceed 4 levels of cross-program invocations. - -Since the compute budget is consumed incrementally as the program executes the -total budget consumption will be a combination of the various costs of the -operations it performs. - -At runtime a program may log how much of the compute budget remains. See -[debugging](developing/deployed-programs/debugging.md#monitoring-compute-budget-consumption) -for more information. - -The budget values are conditional on feature enablement, take a look the compute -budget's -[new](https://github.com/solana-labs/solana/blob/d3a3a7548c857f26ec2cb10e270da72d373020ec/sdk/src/process_instruction.rs#L97) -function to find out how the budget is constructed. An understanding of how -[features](runtime-features.md) work and what features are enabled on the -cluster being used are required to determine the current budget's values. \ No newline at end of file diff --git a/docs/src/developing/programming-model/cpi.md b/docs/src/developing/programming-model/cpi.md deleted file mode 100644 index 1b17ede78..000000000 --- a/docs/src/developing/programming-model/cpi.md +++ /dev/null @@ -1,155 +0,0 @@ ---- -title: Cross-Program Invocation ---- - -## Problem - -In today's implementation, a client can create a transaction that modifies two -accounts, each owned by a separate on-chain program: - -```rust,ignore -let message = Message::new(vec![ - token_instruction::pay(&alice_pubkey), - acme_instruction::launch_missiles(&bob_pubkey), -]); -client.send_and_confirm_message(&[&alice_keypair, &bob_keypair], &message); -``` - -However, the current implementation does not allow the `acme` program to -conveniently invoke `token` instructions on the client's behalf: - -```rust,ignore -let message = Message::new(vec![ - acme_instruction::pay_and_launch_missiles(&alice_pubkey, &bob_pubkey), -]); -client.send_and_confirm_message(&[&alice_keypair, &bob_keypair], &message); -``` - -Currently, there is no way to create instruction `pay_and_launch_missiles` that -executes `token_instruction::pay` from the `acme` program. A possible workaround -is to extend the `acme` program with the implementation of the `token` program -and create `token` accounts with `ACME_PROGRAM_ID`, which the `acme` program is -permitted to modify. With that workaround, `acme` can modify token-like accounts -created by the `acme` program, but not token accounts created by the `token` -program. - -## Solution - -The goal of this design is to modify Solana's runtime such that an on-chain -program can invoke an instruction from another program. - -Given two on-chain programs `token` and `acme`, each implementing instructions -`pay()` and `launch_missiles()` respectively, we would ideally like to implement -the `acme` module with a call to a function defined in the `token` module: - -```rust,ignore -mod acme { - use token; - - fn launch_missiles(accounts: &[AccountInfo]) -> Result<()> { - ... - } - - fn pay_and_launch_missiles(accounts: &[AccountInfo]) -> Result<()> { - token::pay(&accounts[1..])?; - - launch_missiles(accounts)?; - } -``` - -The above code would require that the `token` crate be dynamically linked so -that a custom linker could intercept calls and validate accesses to -`accounts`. Even though the client intends to modify both `token` and -`acme` accounts, only `token` program is permitted to modify the `token` -account, and only the `acme` program is allowed to modify the `acme` account. - -Backing off from that ideal direct cross-program call, a slightly more verbose -solution is to allow `acme` to invoke `token` by issuing a token instruction via -the runtime. - -```rust,ignore -mod acme { - use token_instruction; - - fn launch_missiles(accounts: &[AccountInfo]) -> Result<()> { - ... - } - - fn pay_and_launch_missiles(accounts: &[AccountInfo]) -> Result<()> { - let alice_pubkey = accounts[1].key; - let instruction = token_instruction::pay(&alice_pubkey); - invoke(&instruction, accounts)?; - - launch_missiles(accounts)?; - } -``` - -`invoke()` is built into Solana's runtime and is responsible for routing the -given instruction to the `token` program via the instruction's `program_id` -field. - -Note that `invoke` requires the caller to pass all the accounts required by the -instruction being invoked. This means that both the executable account (the -ones that matches the instruction's program id) and the accounts passed to the -instruction procesor. - -Before invoking `pay()`, the runtime must ensure that `acme` didn't modify any -accounts owned by `token`. It does this by applying the runtime's policy to the -current state of the accounts at the time `acme` calls `invoke` vs. the initial -state of the accounts at the beginning of the `acme`'s instruction. After -`pay()` completes, the runtime must again ensure that `token` didn't modify any -accounts owned by `acme` by again applying the runtime's policy, but this time -with the `token` program ID. Lastly, after `pay_and_launch_missiles()` -completes, the runtime must apply the runtime policy one more time, where it -normally would, but using all updated `pre_*` variables. If executing -`pay_and_launch_missiles()` up to `pay()` made no invalid account changes, -`pay()` made no invalid changes, and executing from `pay()` until -`pay_and_launch_missiles()` returns made no invalid changes, then the runtime -can transitively assume `pay_and_launch_missiles()` as whole made no invalid -account changes, and therefore commit all these account modifications. - -### Instructions that require privileges - -The runtime uses the privileges granted to the caller program to determine what -privileges can be extended to the callee. Privileges in this context refer to -signers and writable accounts. For example, if the instruction the caller is -processing contains a signer or writable account, then the caller can invoke an -instruction that also contains that signer and/or writable account. - -This privilege extension relies on the fact that programs are immutable. In the -case of the `acme` program, the runtime can safely treat the transaction's -signature as a signature of a `token` instruction. When the runtime sees the -`token` instruction references `alice_pubkey`, it looks up the key in the `acme` -instruction to see if that key corresponds to a signed account. In this case, it -does and thereby authorizes the `token` program to modify Alice's account. - -### Program signed accounts - -Programs can issue instructions that contain signed accounts that were not -signed in the original transaction by using [Program derived -addresses](program-derived-addresses.md). - -To sign an account with program derived addresses, a program may -`invoke_signed()`. - -```rust,ignore - invoke_signed( - &instruction, - accounts, - &[&["First addresses seed"], - &["Second addresses first seed", "Second addresses second seed"]], - )?; -``` - -### Call Depth - -Cross-program invocations allow programs to invoke other programs directly but -the depth is constrained currently to 4. - -### Reentrancy - -Reentrancy is currently limited to direct self recursion capped at a fixed -depth. This restriction prevents situations where a program might invoke another -from an intermediary state without the knowledge that it might later be called -back into. Direct recursion gives the program full control of its state at the -point that it gets called back. diff --git a/docs/src/developing/programming-model/overview.md b/docs/src/developing/programming-model/overview.md index aadedb721..eabb9033e 100644 --- a/docs/src/developing/programming-model/overview.md +++ b/docs/src/developing/programming-model/overview.md @@ -2,11 +2,13 @@ title: "Overview" --- -An _app_ interacts with a Solana cluster by sending it _transactions_ with one -or more _instructions_. The Solana _runtime_ passes those instructions to -_programs_ deployed by app developers beforehand. An instruction might, for -example, tell a program to transfer _lamports_ from one _account_ to another or -create an interactive contract that governs how lamports are transferred. +An [app](terminology.md#app) interacts with a Solana cluster by sending it +[transactions](transactions.md) with one or more +[instructions](transactions.md#instructions). The Solana [runtime](runtime.md) +passes those instructions to [programs](terminology.md#program) deployed by app developers +beforehand. An instruction might, for example, tell a program to transfer +[lamports](terminology.md#lamports) from one [account](accounts.md) to another +or create an interactive contract that governs how lamports are transferred. Instructions are executed sequentially and atomically for each transaction. If any instruction is invalid, all account changes in the transaction are discarded. diff --git a/docs/src/developing/programming-model/program-derived-addresses.md b/docs/src/developing/programming-model/program-derived-addresses.md deleted file mode 100644 index 26afcb3d1..000000000 --- a/docs/src/developing/programming-model/program-derived-addresses.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -title: Program Derived Addresses ---- - -## Problem - -Programs cannot generate signatures when issuing instructions to other programs -as defined in the [Cross-Program Invocations](cpi.md) -design. - -The lack of programmatic signature generation limits the kinds of programs that -can be implemented in Solana. A program may be given the authority over an -account and later want to transfer that authority to another. This is impossible -today because the program cannot act as the signer in the transaction that gives -authority. - -For example, if two users want to make a wager on the outcome of a game in -Solana, they must each transfer their wager's assets to some intermediary that -will honor their agreement. Currently, there is no way to implement this -intermediary as a program in Solana because the intermediary program cannot -transfer the assets to the winner. - -This capability is necessary for many DeFi applications since they require -assets to be transferred to an escrow agent until some event occurs that -determines the new owner. - -- Decentralized Exchanges that transfer assets between matching bid and ask - orders. - -- Auctions that transfer assets to the winner. - -- Games or prediction markets that collect and redistribute prizes to the - winners. - -## Solution - -The key to the design is two-fold: - -1. Allow programs to control specific addresses, called program addresses, in - such a way that no external user can generate valid transactions with - signatures for those addresses. - -2. Allow programs to programmatically sign for programa addresses that are - present in instructions invoked via [Cross-Program - Invocations](cpi.md). - -Given the two conditions, users can securely transfer or assign the authority of -on-chain assets to program addresses and the program can then assign that -authority elsewhere at its discretion. - -### Private keys for program addresses - -A Program address does not lie on the ed25519 curve and therefore has no valid -private key associated with it, and thus generating a signature for it is -impossible. While it has no private key of its own, it can be used by a program -to issue an instruction that includes the Program address as a signer. - -### Hash-based generated program addresses - -Program addresses are deterministically derived from a collection of seeds and a -program id using a 256-bit pre-image resistant hash function. Program address -must not lie on the ed25519 curve to ensure there is no associated private key. -During generation an error will be returned if the address is found to lie on -the curve. There is about a 50/50 change of this happening for a given -collection of seeds and program id. If this occurs a different set of seeds or -a seed bump (additional 8 bit seed) can be used to find a valid program address -off the curve. - -Deterministic program addresses for programs follow a similar derivation path as -Accounts created with `SystemInstruction::CreateAccountWithSeed` which is -implemented with `system_instruction::create_address_with_seed`. - -For reference that implementation is as follows: - -```rust,ignore -pub fn create_address_with_seed( - base: &Pubkey, - seed: &str, - program_id: &Pubkey, -) -> Result { - if seed.len() > MAX_ADDRESS_SEED_LEN { - return Err(SystemError::MaxSeedLengthExceeded); - } - - Ok(Pubkey::new( - hashv(&[base.as_ref(), seed.as_ref(), program_id.as_ref()]).as_ref(), - )) -} -``` - -Programs can deterministically derive any number of addresses by using seeds. -These seeds can symbolically identify how the addresses are used. - -From `Pubkey`:: - -```rust,ignore -/// Generate a derived program address -/// * seeds, symbolic keywords used to derive the key -/// * program_id, program that the address is derived for -pub fn create_program_address( - seeds: &[&[u8]], - program_id: &Pubkey, -) -> Result -``` - -### Using program addresses - -Clients can use the `create_program_address` function to generate a destination -address. - -```rust,ignore -// deterministically derive the escrow key -let escrow_pubkey = create_program_address(&[&["escrow"]], &escrow_program_id); - -// construct a transfer message using that key -let message = Message::new(vec![ - token_instruction::transfer(&alice_pubkey, &escrow_pubkey, 1), -]); - -// process the message which transfer one 1 token to the escrow -client.send_and_confirm_message(&[&alice_keypair], &message); -``` - -Programs can use the same function to generate the same address. In the function -below the program issues a `token_instruction::transfer` from a program address -as if it had the private key to sign the transaction. - -```rust,ignore -fn transfer_one_token_from_escrow( - program_id: &Pubkey, - keyed_accounts: &[KeyedAccount] -) -> Result<()> { - - // User supplies the destination - let alice_pubkey = keyed_accounts[1].unsigned_key(); - - // Deterministically derive the escrow pubkey. - let escrow_pubkey = create_program_address(&[&["escrow"]], program_id); - - // Create the transfer instruction - let instruction = token_instruction::transfer(&escrow_pubkey, &alice_pubkey, 1); - - // The runtime deterministically derives the key from the currently - // executing program ID and the supplied keywords. - // If the derived address matches a key marked as signed in the instruction - // then that key is accepted as signed. - invoke_signed(&instruction, &[&["escrow"]])? -} -``` - -### Instructions that require signers - -The addresses generated with `create_program_address` are indistinguishable from -any other public key. The only way for the runtime to verify that the address -belongs to a program is for the program to supply the seeds used to generate the -address. - -The runtime will internally call `create_program_address`, and compare the -result against the addresses supplied in the instruction. diff --git a/docs/src/developing/programming-model/runtime-features.md b/docs/src/developing/programming-model/runtime-features.md deleted file mode 100644 index 4a61f19bf..000000000 --- a/docs/src/developing/programming-model/runtime-features.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -title: "Runtime Features" ---- - -As Solana evolves, new features or patches may be introduced that changes the -behavior of the cluster and how programs run. Changes in behavior must be -coordinated between the various nodes of the cluster, if nodes do not coordinate -then these changes can result in a break-down of consensus. Solana supports a -mechanism called runtime features to facilitate the smooth adoption of changes. - -Runtime features are epoch coordinated events where one or more behavior changes -to the cluster will occur. New changes to Solana that will change behavior are -wrapped with feature gates and disabled by default. The Solana tools are then -used to activate a feature, which marks it pending, once marked pending the -feature will be activated at the next epoch. - -To determine which features are activated use the [Solana command-line -tools](cli/install-solana-cli-tools.md): - -```bash -solana feature status -``` - -If you encounter problems first ensure that the Solana tools version you are -using match the version returned by `solana cluster-version`. If they do not -match [install the correct tool suite](cli/install-solana-cli-tools.md). diff --git a/docs/src/developing/programming-model/runtime.md b/docs/src/developing/programming-model/runtime.md new file mode 100644 index 000000000..cee5d5a6c --- /dev/null +++ b/docs/src/developing/programming-model/runtime.md @@ -0,0 +1,127 @@ +--- +title: "Runtime" +--- + +## Capability of Programs + +The runtime only permits the owner program to debit the account or modify its +data. The program then defines additional rules for whether the client can +modify accounts it owns. In the case of the System program, it allows users to +transfer lamports by recognizing transaction signatures. If it sees the client +signed the transaction using the keypair's _private key_, it knows the client +authorized the token transfer. + +In other words, the entire set of accounts owned by a given program can be +regarded as a key-value store where a key is the account address and value is +program-specific arbitrary binary data. A program author can decide how to +manage the program's whole state as possibly many accounts. + +After the runtime executes each of the transaction's instructions, it uses the +account metadata to verify that the access policy was not violated. If a program +violates the policy, the runtime discards all account changes made by all +instructions in the transaction and marks the transaction as failed. + +### Policy + +After a program has processed an instruction the runtime verifies that the +program only performed operations it was permitted to, and that the results +adhere to the runtime policy. + +The policy is as follows: +- Only the owner of the account may change owner. + - And only if the account is writable. + - And only if the data is zero-initialized or empty. +- An account not assigned to the program cannot have its balance decrease. +- The balance of read-only and executable accounts may not change. +- Only the system program can change the size of the data and only if the system + program owns the account. +- Only the owner may change account data. + - And if the account is writable. + - And if the account is not executable. +- Executable is one-way (false->true) and only the account owner may set it. +- No one modification to the rent_epoch associated with this account. + +## Compute Budget + +To prevent a program from abusing computation resources each instruction in a +transaction is given a compute budget. The budget consists of computation units +that are consumed as the program performs various operations and bounds that the +program may not exceed. When the program consumes its entire budget or exceeds +a bound then the runtime halts the program and returns an error. + +The following operations incur a compute cost: +- Executing BPF instructions +- Calling system calls + - logging + - creating program addresses + - cross-program invocations + - ... + +For cross-program invocations the programs invoked inherit the budget of their +parent. If an invoked program consume the budget or exceeds a bound the entire +invocation chain and the parent are halted. + +The current [compute +budget](https://github.com/solana-labs/solana/blob/d3a3a7548c857f26ec2cb10e270da72d373020ec/sdk/src/process_instruction.rs#L65) +can be found in the Solana SDK. + +For example, if the current budget is: + +```rust +max_units: 200,000, +log_units: 100, +log_u64_units: 100, +create_program address units: 1500, +invoke_units: 1000, +max_invoke_depth: 4, +max_call_depth: 64, +stack_frame_size: 4096, +log_pubkey_units: 100, +``` + +Then the program +- Could execute 200,000 BPF instructions if it does nothing else +- Could log 2,000 log messages +- Can not exceed 4k of stack usage +- Can not exceed a BPF call depth of 64 +- Cannot exceed 4 levels of cross-program invocations. + +Since the compute budget is consumed incrementally as the program executes the +total budget consumption will be a combination of the various costs of the +operations it performs. + +At runtime a program may log how much of the compute budget remains. See +[debugging](developing/deployed-programs/debugging.md#monitoring-compute-budget-consumption) +for more information. + +The budget values are conditional on feature enablement, take a look the compute +budget's +[new](https://github.com/solana-labs/solana/blob/d3a3a7548c857f26ec2cb10e270da72d373020ec/sdk/src/process_instruction.rs#L97) +function to find out how the budget is constructed. An understanding of how +[features](runtime.md#features) work and what features are enabled on the +cluster being used are required to determine the current budget's values. + +## New Features + +As Solana evolves, new features or patches may be introduced that changes the +behavior of the cluster and how programs run. Changes in behavior must be +coordinated between the various nodes of the cluster, if nodes do not coordinate +then these changes can result in a break-down of consensus. Solana supports a +mechanism called runtime features to facilitate the smooth adoption of changes. + +Runtime features are epoch coordinated events where one or more behavior changes +to the cluster will occur. New changes to Solana that will change behavior are +wrapped with feature gates and disabled by default. The Solana tools are then +used to activate a feature, which marks it pending, once marked pending the +feature will be activated at the next epoch. + +To determine which features are activated use the [Solana command-line +tools](cli/install-solana-cli-tools.md): + +```bash +solana feature status +``` + +If you encounter problems first ensure that the Solana tools version you are +using match the version returned by `solana cluster-version`. If they do not +match [install the correct tool suite](cli/install-solana-cli-tools.md). diff --git a/docs/src/developing/programming-model/secpk1-instructions.md b/docs/src/developing/programming-model/secpk1-instructions.md deleted file mode 100644 index a598eab7a..000000000 --- a/docs/src/developing/programming-model/secpk1-instructions.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: secp256k1 builtin instructions ---- - -## Problem - -Performing multiple secp256k1 pubkey recovery operations (ecrecover) in BPF -would exceed the transction bpf instruction limit and even if the limit is -increased it would take a long time to process. ecrecover is an ethereum -instruction which takes a signature and message and recovers a publickey, a -comparison to that public key can thus verify that the signature is valid. - -Since there needs to be 10-20 signatures in the transaction as well as the -signing data which is on the order of 500 bytes, transaction space is a concern. -But also having more concentrated similar work should provide for easier -optimization. - -## Solution - -Add a new builtin instruction which takes in as the first byte a count of the -following struct serialized in the instruction data: - -``` -struct Secp256k1SignatureOffsets { - secp_signature_key_offset: u16, // offset to [signature,recovery_id,etherum_address] of 64+1+20 bytes - secp_signature_instruction_index: u8, // instruction index to find data - secp_pubkey_offset: u16, // offset to [signature,recovery_id] of 64+1 bytes - secp_signature_instruction_index: u8, // instruction index to find data - secp_message_data_offset: u16, // offset to start of message data - secp_message_data_size: u16, // size of message data - secp_message_instruction_index: u8, // index of instruction data to get message data -} -``` - -Pseudo code of the operation: -``` -process_instruction() { - for i in 0..count { - // i'th index values referenced: - instructions = &transaction.message().instructions - signature = instructions[secp_signature_instruction_index].data[secp_signature_offset..secp_signature_offset + 64] - recovery_id = instructions[secp_signature_instruction_index].data[secp_signature_offset + 64] - ref_eth_pubkey = instructions[secp_pubkey_instruction_index].data[secp_pubkey_offset..secp_pubkey_offset + 32] - message_hash = keccak256(instructions[secp_message_instruction_index].data[secp_message_data_offset..secp_message_data_offset + secp_message_data_size]) - pubkey = ecrecover(signature, recovery_id, message_hash) - eth_pubkey = keccak256(pubkey[1..])[12..] - if eth_pubkey != ref_eth_pubkey { - return Error - } - } - return Success -} -``` - -This allows the user to specify any instruction data in the transaction for -signature and message data. By specifying a special instructions sysvar, one can -also receive data from the transaction itself. - -Cost of the transaction will count the number of signatures to verify multiplied -by the signature cost verify multiplier. - -## Optimization notes - -The operation will have to take place after (at least partial) deserialization, -but all inputs come from the transaction data itself, this allows it to be -relatively easy to execute in parallel to transaction processing and PoH -verification. - -## Other solutions - -* Instruction available as CPI such that the program can call as desired or a - syscall which can operate on the instruction inline. - - Could be harder to optimize given that it generally either requires bpf - program scan to determine the inputs to the operation, or the - implementation needs to just wait until the program hits the operation in - bpf processing to evaluate it. - - Vector version of the operation could allow for somewhat efficient simd/gpu - execution. For most efficient though, batching with other instructions in - the pipeline would be ideal. - - Pros - Nicer interface for the user. - -* Async execution environment inside bpf - - Might be hard to optimize for devices like gpus which cannot queue work for - itself easily - - Might be easier to optimize on cpu since ordering can be more explicit - -* All inputs have to come from the instruction - - Pros - easier to optimize, data is already sent to the GPU for instance for - regular sigverify. Probably still need to wait for deserialize though. - - Cons - ask for pubkeys outside the transaction data itself since they would - not be stored on the transaction sending client, and larger transaction - size. diff --git a/docs/src/developing/programming-model/transactions.md b/docs/src/developing/programming-model/transactions.md index 9254b4626..cf5eb8ad7 100644 --- a/docs/src/developing/programming-model/transactions.md +++ b/docs/src/developing/programming-model/transactions.md @@ -7,8 +7,81 @@ submitted to the cluster. The Solana runtime will execute a program to process each of the [instructions](terminology.md#instruction) contained in the transaction, in order, and atomically. -See [Anatomy of a Transaction](transaction.md) for more information about how a -transaction is encoded. +## Anatomy of a Transaction + +This section covers the binary format of a transaction. + +### Transaction Format + +A transaction contains a [compact-array](#compact-array-format) of signatures, +followed by a [message](#message-format). Each item in the signatures array is +a [digital signature](#signature-format) of the given message. The Solana +runtime verifies that the number of signatures matches the number in the first +8 bits of the [message header](#message-header-format). It also verifies that +each signature was signed by the private key corresponding to the public key at +the same index in the message's account addresses array. + +#### Signature Format + +Each digital signature is in the ed25519 binary format and consumes 64 bytes. + +### Message Format + +A message contains a [header](#message-header-format), followed by a +compact-array of [account addresses](#account-addresses-format), followed by a +recent [blockhash](#blockhash-format), followed by a compact-array of +[instructions](#instruction-format). + +#### Message Header Format + +The message header contains three unsigned 8-bit values. The first value is the +number of required signatures in the containing transaction. The second value +is the number of those corresponding account addresses that are read-only. The +third value in the message header is the number of read-only account addresses +not requiring signatures. + +#### Account Addresses Format + +The addresses that require signatures appear at the beginning of the account +address array, with addresses requesting write access first and read-only +accounts following. The addresses that do not require signatures follow the +addresses that do, again with read-write accounts first and read-only accounts +following. + +#### Blockhash Format + +A blockhash contains a 32-byte SHA-256 hash. It is used to indicate when a +client last observed the ledger. Validators will reject transactions when the +blockhash is too old. + +### Instruction Format + +An instruction contains a program id index, followed by a compact-array of +account address indexes, followed by a compact-array of opaque 8-bit data. The +program id index is used to identify an on-chain program that can interpret the +opaque data. The program id index is an unsigned 8-bit index to an account +address in the message's array of account addresses. The account address +indexes are each an unsigned 8-bit index into that same array. + +### Compact-Array Format + +A compact-array is serialized as the array length, followed by each array item. +The array length is a special multi-byte encoding called compact-u16. + +#### Compact-u16 Format + +A compact-u16 is a multi-byte encoding of 16 bits. The first byte contains the +lower 7 bits of the value in its lower 7 bits. If the value is above 0x7f, the +high bit is set and the next 7 bits of the value are placed into the lower 7 +bits of a second byte. If the value is above 0x3fff, the high bit is set and +the remaining 2 bits of the value are placed into the lower 2 bits of a third +byte. + +### Account Address Format + +An account address is 32-bytes of arbitrary data. When the address requires a +digital signature, the runtime interprets it as the public key of an ed25519 +keypair. ## Instructions @@ -53,14 +126,22 @@ Which can be found here: https://github.com/solana-labs/solana/blob/6606590b8132e56dab9e60b3f7d20ba7412a736c/sdk/program/src/system_instruction.rs#L220 -### Program ID +### Program Id The instruction's [program id](terminology.md#program-id) specifies which -program will process this instruction. The program's account data contains -information about how the runtime should execute the program, in the case of BPF -programs, the account data holds the BPF bytecode. Program accounts are marked -as executable once they are successfully deployed. The runtime will reject -transactions that specify programs that are not executable. +program will process this instruction. The program's account's owner specifies +which loader should be used to load and execute the program and the data +contains information about how the runtime should execute the program. + +In the case of [deployed BPF +programs](developing/deployed-programs/overview.md), the owner is the BPF Loader +and the account data holds the BPF bytecode. Program accounts are permanently +marked as executable by the loader once they are successfully deployed. The +runtime will reject transactions that specify programs that are not executable. + + +Unlike deployed programs, [builtins](developing/builtins/programs.md) are handled +differently in that they are built directly into the Solana runtime. ### Accounts diff --git a/docs/src/integrations/exchange.md b/docs/src/integrations/exchange.md index f1be6db90..c4c481218 100644 --- a/docs/src/integrations/exchange.md +++ b/docs/src/integrations/exchange.md @@ -382,9 +382,10 @@ For greater flexibility, you can submit withdrawal transfers asynchronously. In these cases, it is your responsibility to verify that the transaction succeeded and was finalized by the cluster. -**Note:** Each transaction contains a [recent blockhash](../transaction.md#blockhash-format) -to indicate its liveness. It is **critical** to wait until this blockhash -expires before retrying a withdrawal transfer that does not appear to have been +**Note:** Each transaction contains a [recent +blockhash](developing/programming-model/transactions.md#blockhash-format) to +indicate its liveness. It is **critical** to wait until this blockhash expires +before retrying a withdrawal transfer that does not appear to have been confirmed or finalized by the cluster. Otherwise, you risk a double spend. See more on [blockhash expiration](#blockhash-expiration) below. diff --git a/docs/src/offline-signing/durable-nonce.md b/docs/src/offline-signing/durable-nonce.md index 15972e103..afb136b6f 100644 --- a/docs/src/offline-signing/durable-nonce.md +++ b/docs/src/offline-signing/durable-nonce.md @@ -3,7 +3,7 @@ title: Durable Transaction Nonces --- Durable transaction nonces are a mechanism for getting around the typical -short lifetime of a transaction's [`recent_blockhash`](../transaction.md#recent-blockhash). +short lifetime of a transaction's [`recent_blockhash`](developing/programming-model/transactions.md#recent-blockhash). They are implemented as a Solana Program, the mechanics of which can be read about in the [proposal](../implemented-proposals/durable-tx-nonces.md). diff --git a/docs/src/transaction.md b/docs/src/transaction.md deleted file mode 100644 index 84b358ef3..000000000 --- a/docs/src/transaction.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Anatomy of a Transaction ---- - -This section documents the binary format of a transaction. - -## Transaction Format - -A transaction contains a [compact-array](#compact-array-format) of signatures, -followed by a [message](#message-format). Each item in the signatures array is -a [digital signature](#signature-format) of the given message. The Solana -runtime verifies that the number of signatures matches the number in the first -8 bits of the [message header](#message-header-format). It also verifies that -each signature was signed by the private key corresponding to the public key at -the same index in the message's account addresses array. - -### Signature Format - -Each digital signature is in the ed25519 binary format and consumes 64 bytes. - -## Message Format - -A message contains a [header](#message-header-format), followed by a -compact-array of [account addresses](#account-addresses-format), followed by a -recent [blockhash](#blockhash-format), followed by a compact-array of -[instructions](#instruction-format). - -### Message Header Format - -The message header contains three unsigned 8-bit values. The first value is the -number of required signatures in the containing transaction. The second value -is the number of those corresponding account addresses that are read-only. The -third value in the message header is the number of read-only account addresses -not requiring signatures. - -### Account Addresses Format - -The addresses that require signatures appear at the beginning of the account -address array, with addresses requesting write access first and read-only -accounts following. The addresses that do not require signatures follow the -addresses that do, again with read-write accounts first and read-only accounts -following. - -### Blockhash Format - -A blockhash contains a 32-byte SHA-256 hash. It is used to indicate when a -client last observed the ledger. Validators will reject transactions when the -blockhash is too old. - -## Instruction Format - -An instruction contains a program ID index, followed by a compact-array of -account address indexes, followed by a compact-array of opaque 8-bit data. The -program ID index is used to identify an on-chain program that can interpret the -opaque data. The program ID index is an unsigned 8-bit index to an account -address in the message's array of account addresses. The account address -indexes are each an unsigned 8-bit index into that same array. - -## Compact-Array Format - -A compact-array is serialized as the array length, followed by each array item. -The array length is a special multi-byte encoding called compact-u16. - -### Compact-u16 Format - -A compact-u16 is a multi-byte encoding of 16 bits. The first byte contains the -lower 7 bits of the value in its lower 7 bits. If the value is above 0x7f, the -high bit is set and the next 7 bits of the value are placed into the lower 7 -bits of a second byte. If the value is above 0x3fff, the high bit is set and -the remaining 2 bits of the value are placed into the lower 2 bits of a third -byte. - -## Account Address Format - -An account address is 32-bytes of arbitrary data. When the address requires a -digital signature, the runtime interprets it as the public key of an ed25519 -keypair.