diff --git a/.gitignore b/.gitignore index fcceba9c53..9d970c4482 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ Cargo.lock /target/ +/doc/book/ **/*.rs.bk .cargo diff --git a/ci/test-stable.sh b/ci/test-stable.sh index 5d2b219697..615836b5d5 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -11,6 +11,18 @@ _() { "$@" } +maybe_install() { + for cmd in "$@"; do + set +e + "$cmd" --help > /dev/null 2>&1 + declare exitcode=$? + set -e + if [[ $exitcode -ne 0 ]]; then + _ cargo install "$cmd" + fi + done +} + _ cargo fmt -- --check _ cargo build --verbose _ cargo test --verbose --lib @@ -33,6 +45,14 @@ for program in programs/native/*; do ) done +# Build the HTML +export PATH=$CARGO_HOME/bin:$PATH +maybe_install mdbook +_ cd doc +_ mdbook test +_ mdbook build +_ cd - + echo --- ci/localnet-sanity.sh ( set -x diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000000..fec26fb0c2 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,25 @@ +## Building the Solana book + +Install mdbook: + +```bash +cargo install mdbook +``` + +Run any Rust tests in the markdown: + +```bash +mdbook test +``` + +Render markdown as HTML: + +```bash +mdbook build +``` + +Render and view the book: + +```bash +mdbook build --open +``` diff --git a/doc/consensus.msc b/doc/art/consensus.msc similarity index 100% rename from doc/consensus.msc rename to doc/art/consensus.msc diff --git a/doc/art/fullnode.bob b/doc/art/fullnode.bob new file mode 100644 index 0000000000..50d4e277f2 --- /dev/null +++ b/doc/art/fullnode.bob @@ -0,0 +1,46 @@ + .---------------------. + | Leader | + | | + .--------. | .----------------. | + | |---->| | | + | Client | | | JsonRpcService | | + | |<----| | | + `----+---` | `----------------` | + | | ^ | + | | | | + | | .--+---. | + | | | Bank | | + | | `------` | + | | ^ | + | | | | .------------. + | | .--+--. .-----. | | | + `-------->| Tpu +-->| Ncp +------>| Validators | + | `-----` `-----` | | | + | | `------------` + `---------------------` + + .-------------------------------. + | Validator | + | | + .--------. | .----------------. | + | |-------------->| | | + | Client | | | JsonRpcService | | + | |<--------------| | | + `--------` | `----------------` | + | ^ | + | | | + | .--+---. | + | | Bank | | + | `------` | + | ^ | + .--------. | | | .------------. + | | | .--+--. | | | + | Leader |<------------->| Tvu +<--------------->| | + | | | `-----` | | Validators | + | | | ^ | | | + | | | | | | | + | | | .--+--. | | | + | |<------------->| Ncp +<--------------->| | + | | | `-----` | | | + `--------` | | `------------` + `-------------------------------` diff --git a/doc/art/lamport.bob b/doc/art/lamport.bob new file mode 100644 index 0000000000..ef61ca17cc --- /dev/null +++ b/doc/art/lamport.bob @@ -0,0 +1,9 @@ + ++-----------+ +-------------+ +--------------+ +--------------------+ +| sigverify |--->| lock memory |--->| validate fee |--->| allocate accounts |---> ++-----------+ +-------------+ +--------------+ +--------------------+ + + +------------+ +---------+ +-=------------+ +--------------+ +--->| load data |--->| execute |--->| commit data |-->|unlock memory | + +------------+ +---------+ +--------------+ +--------------+ + diff --git a/doc/art/sdk-tools.bob b/doc/art/sdk-tools.bob new file mode 100644 index 0000000000..5bbac7146b --- /dev/null +++ b/doc/art/sdk-tools.bob @@ -0,0 +1,24 @@ + +---------------------+ +---------------------+ + | | | | + | +------------+ | | +------------+ | + | | | | | | | | + | | frontend | | | | verifier | | + | | | | | | | | + | +-----+------+ | | +-----+------+ | + | | | | | | + | | | | | | + | +-----+------+ | | +-----+------+ | + | | | | | | | | + | | llvm | | | | loader | | + | | | +------>+ | | | + | +-----+------+ | | +-----+------+ | + | | | | | | + | | | | | | + | +-----+------+ | | +-----+------+ | + | | | | | | | | + | | ELF | | | | runtime | | + | | | | | | | | + | +------------+ | | +------------+ | + | | | | + | client | | solana | + +---------------------+ +---------------------+ diff --git a/doc/art/tpu.bob b/doc/art/tpu.bob new file mode 100644 index 0000000000..cf046ecfb4 --- /dev/null +++ b/doc/art/tpu.bob @@ -0,0 +1,21 @@ + .----------------------------------------------------. + | Tpu .------------. | + | | PohService | | + | `-------+----` | + | ^ | | + | | v | + | .-------. .-----------. .-+-------. .-------. | + .---------. | | Fetch | | SigVerify | | Banking | | Write | | .------------. + | Clients |--->| Stage |->| Stage |->| Stage |-->| Stage +--->| Validators | + `---------` | | | | | | | | | | `------------` + | `-------` `-----------` `----+----` `---+---` | + | | | | + | | | | + | | | | + | | | | + `---------------------------------|------------|-----` + | | + v v + .------. .--------. + | Bank | | Ledger | + `------` `--------` diff --git a/doc/art/tvu.bob b/doc/art/tvu.bob new file mode 100644 index 0000000000..942979f215 --- /dev/null +++ b/doc/art/tvu.bob @@ -0,0 +1,23 @@ + .------------------------------------------------. + | | + | .------------------------------------+------------. + | | Tvu | | + | | | | + | | | | .------------. + | | .----------------+-------------->| Validators | + v | .-------. | | | `------------` + .----+---. | | | .----+-------. .----+---------. | + | Leader |--------->| Blob | | Retransmit | | Replicate | | + `--------` | | Fetch |-->| Stage |-->| Stage / | | + .------------. | | Stage | | | | Vote Stage | | + | Validators |----->| | `------------` `----+---------` | + `------------` | `-------` | | + | | | + | | | + | | | + `------------------------------------|------------` + | + v + .------. + | Bank | + `------` diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md new file mode 100644 index 0000000000..d33ef7fa01 --- /dev/null +++ b/doc/src/SUMMARY.md @@ -0,0 +1,27 @@ +# Solana Architecture + +- [Introduction](ch00-00-introduction.md) + +- [Synchronization](ch01-00-synchronization.md) + - [Introduction to VDFs](ch01-01-vdf.md) + - [Proof of History](ch01-02-poh.md) + +- [Fullnode](ch02-00-fullnode.md) + - [Tpu](ch02-02-tpu.md) + - [Tvu](ch02-03-tvu.md) + - [Ncp](ch02-04-ncp.md) + - [JsonRpcService](ch02-05-jsonrpc-service.md) + +- [Avalanche replication](ch03-00-avalanche.md) + - [Proof of replication](ch03-01-porep.md) + +- [On-chain programs](ch04-00-scaling-programs.md) + - [The Runtime](ch04-01-runtime.md) + - [Ledger format](ch04-02-ledger.md) + +## Appendix + +- [Appendix](appendix-00.md) + - [Terminology](appendix-01-terminology.md) + - [JSON RPC API](appendix-02-jsonrpc-api.md) + diff --git a/doc/src/appendix-00.md b/doc/src/appendix-00.md new file mode 100644 index 0000000000..62910e6484 --- /dev/null +++ b/doc/src/appendix-00.md @@ -0,0 +1,4 @@ +# Appendix + +The following sections contain reference material you may find useful in your +Solana journey. diff --git a/doc/src/appendix-01-terminology.md b/doc/src/appendix-01-terminology.md new file mode 100644 index 0000000000..b627f768fe --- /dev/null +++ b/doc/src/appendix-01-terminology.md @@ -0,0 +1,33 @@ +## Appendix A: Terminology + +### Teminology Currently in Use + +The following list contains words commonly used throughout the Solana architecture. + +* account - a persistent file addressed by pubkey and with tokens tracking its lifetime +* cluster - a set of fullnodes maintaining a single ledger +* finality - the wallclock duration between a leader creating a tick entry and recoginizing + a supermajority of validator votes with a ledger interpretation that matches the leader's +* fullnode - a full participant in the cluster - either a leader or validator node +* entry - an entry on the ledger - either a tick or a transactions entry +* instruction - the smallest unit of a program that a client can include in a transaction +* keypair - a public and secret key +* mips - millions of instructions per second +* node count - the number of fullnodes participating in a cluster +* program - the code that interprets instructions +* pubkey - the public key of a keypair +* tick - a ledger entry that estimates wallclock duration +* tick height - the Nth tick in the ledger +* tps - transactions per second +* transaction - one or more instructions signed by the client and executed atomically +* transactions entry - a set of transactions that may be executed in parallel + + +### Terminology Reserved for Future Use + +The following keywords do not have any functionality but are reserved by Solana +for potential future use. + +* mips - millions of instructions per second +* public key - We currently use `pubkey` +* secret key - Users currently only use `keypair` diff --git a/doc/json-rpc.md b/doc/src/appendix-02-jsonrpc-api.md similarity index 99% rename from doc/json-rpc.md rename to doc/src/appendix-02-jsonrpc-api.md index 9c54d79e7b..d56b04eb25 100644 --- a/doc/json-rpc.md +++ b/doc/src/appendix-02-jsonrpc-api.md @@ -1,4 +1,4 @@ -Solana JSON RPC API +JSON RPC API === Solana nodes accept HTTP requests using the [JSON-RPC 2.0](https://www.jsonrpc.org/specification) specification. diff --git a/doc/src/ch00-00-introduction.md b/doc/src/ch00-00-introduction.md new file mode 100644 index 0000000000..4b11047b6a --- /dev/null +++ b/doc/src/ch00-00-introduction.md @@ -0,0 +1,31 @@ +# Introduction + +This document defines the architecture of Solana, a blockchain built from the +ground up for scale. The goal of the architecture is to demonstrate there +exists a set of software algorithms that in combination, removes software as a +performance bottleneck, allowing transaction throughput to scale proportionally +with network bandwidth. The architecture goes on to satisfy all three desirable +properties of a proper blockchain, that it not only be scalable, but that it is +also secure and decentralized. + +With this architecture, we calculate a theoretical upper bound of 710 thousand +transactions per second (tps) on a standard gigabit network and 28.4 million +tps on 40 gigabit. In practice, our focus has been on gigabit. We soak-tested +a 150 node permissioned testnet and it is able to maintain a mean transaction +throughput of approximately 200 thousand tps with peaks over 400 thousand. + +Furthermore, we have found high throughput extends beyond simple payments, and +that this architecture is also able to perform safe, concurrent execution of +programs authored in a general purpose programming language, such as C. We feel +the extension warrants industry focus on an additional performance metric +already common in the CPU industry, millions of *instructions* per second or +mips. By measuring mips, we see that batching instructions within a transaction +amortizes the cost of signature verification, lifting the maximum theoretical +instruction throughput up to almost exactly that of centralized databases. + +Lastly, we discuss the relationships between high throughput, security and +transaction fees. Solana's efficient use hardware drives transaction fees into +the ballpark of 1/1000th of a cent. The drop in fees in turn makes certain +denial of service attacks cheaper. We discuss what these attacks look like and +Solana's techniques to defend against them. + diff --git a/doc/src/ch01-00-synchronization.md b/doc/src/ch01-00-synchronization.md new file mode 100644 index 0000000000..f568d4845a --- /dev/null +++ b/doc/src/ch01-00-synchronization.md @@ -0,0 +1,38 @@ +# Synchronization + +It's possible for a centralized database to process 710,000 transactions per +second on a standard gigabit network if the transactions are, on average, no +more than 176 bytes. A centralized database can also replicate itself and +maintain high availability without significantly compromising that transaction +rate using the distributed system technique known as Optimistic Concurrency +Control [\[H.T.Kung, J.T.Robinson +(1981)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.65.4735). At +Solana, we're demonstrating that these same theoretical limits apply just as +well to blockchain on an adversarial network. The key ingredient? Finding a way +to share time when nodes can't trust one-another. Once nodes can trust time, +suddenly ~40 years of distributed systems research becomes applicable to +blockchain! + +> Perhaps the most striking difference between algorithms obtained by our +> method and ones based upon timeout is that using timeout produces a +> traditional distributed algorithm in which the processes operate +> asynchronously, while our method produces a globally synchronous one in which +> every process does the same thing at (approximately) the same time. Our +> method seems to contradict the whole purpose of distributed processing, which +> is to permit different processes to operate independently and perform +> different functions. However, if a distributed system is really a single +> system, then the processes must be synchronized in some way. Conceptually, +> the easiest way to synchronize processes is to get them all to do the same +> thing at the same time. Therefore, our method is used to implement a kernel +> that performs the necessary synchronization--for example, making sure that +> two different processes do not try to modify a file at the same time. +> Processes might spend only a small fraction of their time executing the +> synchronizing kernel; the rest of the time, they can operate +> independently--e.g., accessing different files. This is an approach we have +> advocated even when fault-tolerance is not required. The method's basic +> simplicity makes it easier to understand the precise properties of a system, +> which is crucial if one is to know just how fault-tolerant the system is. +> [\[L.Lamport +> (1984)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1078) + + diff --git a/doc/src/ch01-01-vdf.md b/doc/src/ch01-01-vdf.md new file mode 100644 index 0000000000..5188950d9d --- /dev/null +++ b/doc/src/ch01-01-vdf.md @@ -0,0 +1,18 @@ +# Introduction to VDFs + +A Verifiable Delay Function is conceptually a water clock where its water marks +can be recorded and later verified that the water most certainly passed +through. Anatoly describes the water clock analogy in detail here: + +[water clock analogy](https://medium.com/solana-labs/proof-of-history-explained-by-a-water-clock-e682183417b8) + +The same technique has been used in Bitcoin since day one. The Bitcoin feature +is called nLocktime and it can be used to postdate transactions using block +height instead of a timestamp. As a Bitcoin client, you'd use block height +instead of a timestamp if you don't trust the network. Block height turns out +to be an instance of what's being called a Verifiable Delay Function in +cryptography circles. It's a cryptographically secure way to say time has +passed. In Solana, we use a far more granular verifiable delay function, a SHA +256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we +implement Optimistic Concurrency Control and are now well en route towards that +theoretical limit of 710,000 transactions per second. diff --git a/doc/src/ch01-02-poh.md b/doc/src/ch01-02-poh.md new file mode 100644 index 0000000000..b76776fd4c --- /dev/null +++ b/doc/src/ch01-02-poh.md @@ -0,0 +1,43 @@ +# Proof of History + +[Proof of History overview](https://medium.com/solana-labs/proof-of-history-a-clock-for-blockchain-cf47a61a9274) + +## Relationship to consensus mechanisms + +Most confusingly, a Proof of History (PoH) is more similar to a Verifiable +Delay Function (VDF) than a Proof of Work or Proof of Stake consensus +mechanism. The name unfortunately requires some historical context to +understand. Proof of History was developed by Anatoly Yakovenko in November of +2017, roughly 6 months before we saw a [paper using the term +VDF](https://eprint.iacr.org/2018/601.pdf). At that time, it was commonplace to +publish new proofs of some desirable property used to build most any blockchain +component. Some time shortly after, the crypto community began charting out all +the different consensus mechanisms and because most of them started with "Proof +of", the prefix became synonymous with a "consensus" suffix. Proof of History +is not a consensus mechanism, but it is used to improve the performance of +Solana's Proof of Stake consensus. It is also used to improve the performance +of the replication and storage protocols. To minimize confusion, Solana may +rebrand PoH to some flavor of the term VDF. + + +## Relationship to VDFs + +A desirable property of a VDF is that verification time is very fast. Solana's +approach to verifying its delay function is proportional to the time it took to +create it. Split over a 4000 core GPU, it is sufficiently fast for Solana's +needs, but if you asked the authors the paper cited above, they might tell you +(and have) that Solana's approach is algorithmically slow it shouldn't be +called a VDF. We argue the term VDF should represent the category of verifiable +delay functions and not just the subset with certain performance +characteristics. Until that's resolved, Solana will likely continue using the +term PoH for its application-specific VDF. + +Another difference between PoH and VDFs used only for tracking duration, is +that PoH's hash chain includes hashes of any data the application observed. +That data is a double-edged sword. On one side, the data "proves history" - +that the data most certainly existed before hashes after it. On the side, it +means the application can manipulate the hash chain by changing *when* the data +is hashed. The PoH chain therefore does not serve as a good source of +randomness whereas a VDF without that data could. Solana's leader selection +algorithm (TODO: add link), for example, is derived only from the VDF *height* +and not its hash at that height. diff --git a/doc/src/ch02-00-fullnode.md b/doc/src/ch02-00-fullnode.md new file mode 100644 index 0000000000..9e5f104746 --- /dev/null +++ b/doc/src/ch02-00-fullnode.md @@ -0,0 +1,7 @@ +# Fullnode + +Fullnode block diagrams + +## Pipelining + +## Pipeline Stages diff --git a/doc/src/ch02-02-tpu.md b/doc/src/ch02-02-tpu.md new file mode 100644 index 0000000000..5e5a69364a --- /dev/null +++ b/doc/src/ch02-02-tpu.md @@ -0,0 +1,5 @@ +# The Tpu + +Tpu block diagram + +The Transaction Processing Unit diff --git a/doc/src/ch02-03-tvu.md b/doc/src/ch02-03-tvu.md new file mode 100644 index 0000000000..225fbab9d3 --- /dev/null +++ b/doc/src/ch02-03-tvu.md @@ -0,0 +1,5 @@ +# Tvu + +Tvu block diagram + +The Transaction Validation Unit diff --git a/doc/src/ch02-04-ncp.md b/doc/src/ch02-04-ncp.md new file mode 100644 index 0000000000..8c08052f3d --- /dev/null +++ b/doc/src/ch02-04-ncp.md @@ -0,0 +1,3 @@ +# Ncp + +The Network Control Plane implements a gossip network between all nodes on in the cluster. diff --git a/doc/src/ch02-05-jsonrpc-service.md b/doc/src/ch02-05-jsonrpc-service.md new file mode 100644 index 0000000000..835dec81ce --- /dev/null +++ b/doc/src/ch02-05-jsonrpc-service.md @@ -0,0 +1 @@ +# JsonRpcService diff --git a/doc/src/ch03-00-avalanche.md b/doc/src/ch03-00-avalanche.md new file mode 100644 index 0000000000..7c23bc545a --- /dev/null +++ b/doc/src/ch03-00-avalanche.md @@ -0,0 +1,22 @@ +# Avalanche replication + +The [Avalance explainer video](https://www.youtube.com/watch?v=qt_gDRXHrHQ) is +a conceptual overview of how a Solana leader can continuously process a gigabit +of transaction data per second and then get that same data, after being +recorded on the ledger, out to multiple validators on a single gigabit +backplane. + +In practice, we found that just one level of the Avalanche validator tree is +sufficient for at least 150 validators. We anticipate adding the second level +to solve one of two problems: + +1. To transmit ledger segments to slower "replicator" nodes. +2. To scale up the number of validators nodes. + +Both problems justify the additional level, but you won't find it implemented +in the reference design just yet, because Solana's gossip implementation is +currently the bottleneck on the number of nodes per Solana cluster. That work +is being actively developed here: + +[Scalable Gossip](https://github.com/solana-labs/solana/pull/1546) + diff --git a/doc/src/ch03-01-porep.md b/doc/src/ch03-01-porep.md new file mode 100644 index 0000000000..176b740ebc --- /dev/null +++ b/doc/src/ch03-01-porep.md @@ -0,0 +1 @@ +# Proof of replication diff --git a/doc/src/ch04-00-scaling-programs.md b/doc/src/ch04-00-scaling-programs.md new file mode 100644 index 0000000000..334927231e --- /dev/null +++ b/doc/src/ch04-00-scaling-programs.md @@ -0,0 +1,26 @@ +# The LAMPORT execution environment + +## Introduction + +With LAMPORT (Language-Agnostic, Memory-oriented, Parallel-friendly, Optimized +Run-Time), we can execute smart contracts concurrently, and written in the +client’s choice of programming language. Furthermore, we demonstrate Solana’s +built-in smart contract language Budget can target LAMPORT without any loss in +performance. The two features that allow LAMPORT to work: + +Client-owned memory identified by public keys. By declaring ownership upfront +and separating the program’s state from the program, the runtime knows which +contracts can safely be executed concurrently. Solana’s blockchain-encoded VDF +tells validator nodes at precisely what times they need to end up in the same +state. Between those times, they are free to introduce non-deterministic +behavior as-needed to improve execution times. + +## Toolchain Stack + +SDK tools + +As shown in the diagram above an untrusted client, creates a program in the +front-end language of her choice, (like C/C++/Rust/Lua), and compiles it with +LLVM to a position independent shared object ELF, targeting BPF bytecode. +Solana will safely load and execute the ELF. + diff --git a/doc/src/ch04-01-runtime.md b/doc/src/ch04-01-runtime.md new file mode 100644 index 0000000000..b74f04bc42 --- /dev/null +++ b/doc/src/ch04-01-runtime.md @@ -0,0 +1,94 @@ +# Runtime + +The goal with the runtime is to have a general purpose execution environment +that is highly parallelizable. To achieve this goal the runtime forces each +Instruction to specify all of its memory dependencies up front, and therefore a +single Instruction cannot cause a dynamic memory allocation. An explicit +Instruction for memory allocation from the `SystemProgram::CreateAccount` is +the only way to allocate new memory in the engine. A Transaction may compose +multiple Instruction, including `SystemProgram::CreateAccount`, into a single +atomic sequence which allows for memory allocation to achieve a result that is +similar to dynamic allocation. + + +### State + +State is addressed by an Account which is at the moment simply the Pubkey. Our +goal is to eliminate memory allocation from within the program itself. Thus +the client of the program provides all the state that is necessary for the +program to execute in the transaction itself. The runtime interacts with the +program through an entry point with a well defined interface. The userdata +stored in an Account is an opaque type to the runtime, a `Vec`, the +contents of which the program code has full control over. + +The Transaction structure specifies a list of Pubkey's and signatures for those +keys and a sequential list of instructions that will operate over the state's +associated with the `account_keys`. For the transaction to be committed all +the instructions must execute successfully, if any abort the whole transaction +fails to commit. + +### Account structure Accounts maintain token state as well as program specific +memory. + +# Transaction Engine + +At its core, the engine looks up all the Pubkeys maps them to accounts and +routs them to the `program_id` entry point. + +## Execution + +Transactions are batched and processed in a pipeline + +LAMPORT pipeline + +At the `execute` stage, the loaded pages have no data dependencies, so all the +programs can be executed in parallel. + +The runtime enforces the following rules: + +1. The `program_id` code is the only code that will modify the contents of + `Account::userdata` of Account's that have been assigned to it. This means +that upon assignment userdata vector is guaranteed to be `0`. +2. Total balances on all the accounts is equal before and after execution of a + Transaction. +3. Balances of each of the accounts not assigned to `program_id` must be equal + to or greater after the Transaction than before the transaction. +4. All Instructions in the Transaction executed without a failure. + +## Entry Point Execution of the program involves mapping the Program's public +key to an entry point which takes a pointer to the transaction, and an array of +loaded pages. + +## System Interface + +The interface is best described by the `Instruction::userdata` that the +user encodes. +* `CreateAccount` - This allows the user to create and assign an Account to a + Program. +* `Assign` - allows the user to assign an existing account to a `Program`. +* `Move` - moves tokens between `Account`s that are associated with + `SystemProgram`. This cannot be used to move tokens of other `Account`s. +Programs need to implement their own version of Move. + +## Notes + +1. There is no dynamic memory allocation. Client's need to call the +`SystemProgram` to create memory before passing it to another program. This +Instruction can be composed into a single Transaction with the call to the +program itself. +2. Runtime guarantees that when memory is assigned to the `Program` it is zero +initialized. +3. Runtime guarantees that `Program`'s code is the only thing that can modify +memory that its assigned to +4. Runtime guarantees that the `Program` can only spend tokens that are in +`Account`s that are assigned to it +5. Runtime guarantees the balances belonging to `Account`s are balanced before +and after the transaction +6. Runtime guarantees that multiple instructions all executed successfully when +a transaction is committed. + +# Future Work + +* [Continuations and Signals for long running + Transactions](https://github.com/solana-labs/solana/issues/1485) + diff --git a/doc/src/ch04-02-ledger.md b/doc/src/ch04-02-ledger.md new file mode 100644 index 0000000000..a24627dfc1 --- /dev/null +++ b/doc/src/ch04-02-ledger.md @@ -0,0 +1 @@ +# Ledger format diff --git a/doc/src/img/fullnode.svg b/doc/src/img/fullnode.svg new file mode 100644 index 0000000000..874fb71e96 --- /dev/null +++ b/doc/src/img/fullnode.svg @@ -0,0 +1,475 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Client + + + + +Client + + + + +Leader + + + + +Leader + + + + +Validator + + + + +JsonRpcService + + + + +Bank + + + + +Tpu + + + + +Ncp + + + + +JsonRpcService + + + + +Bank + + + + +Tvu + + + + +Ncp + + + + +Validators + + + + +Validators + + + diff --git a/doc/src/img/lamport.svg b/doc/src/img/lamport.svg new file mode 100644 index 0000000000..3e823946d9 --- /dev/null +++ b/doc/src/img/lamport.svg @@ -0,0 +1,211 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +sigverify + + + + +load + + + + +data + + + + +lock + + + + +memory + + + + +execute + + + + +validate + + + + +commit + + + + +data + + + + +fee + + + + +unlock + + + + +allocate + + + + +memory + + + + +accounts + + + diff --git a/doc/src/img/sdk-tools.svg b/doc/src/img/sdk-tools.svg new file mode 100644 index 0000000000..e9f946986f --- /dev/null +++ b/doc/src/img/sdk-tools.svg @@ -0,0 +1,164 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +frontend + + + + +llvm + + + + +ELF + + + + +client + + + + +verifier + + + + +loader + + + + +runtime + + + + +solana + + + diff --git a/doc/src/img/tpu.svg b/doc/src/img/tpu.svg new file mode 100644 index 0000000000..794b8c9cd2 --- /dev/null +++ b/doc/src/img/tpu.svg @@ -0,0 +1,323 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Clients + + + + +Tpu + + + + +Fetch + + + + +Stage + + + + +SigVerify + + + + +Stage + + + + +PohService + + + + +Banking + + + + +Stage + + + + +Bank + + + + +Write + + + + +Stage + + + + +Ledger + + + + +Validators + + + diff --git a/doc/src/img/tvu.svg b/doc/src/img/tvu.svg new file mode 100644 index 0000000000..75ddbaeff8 --- /dev/null +++ b/doc/src/img/tvu.svg @@ -0,0 +1,306 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Leader + + + + +Validators + + + + +Tvu + + + + +Blob + + + + +Fetch + + + + +Stage + + + + +Retransmit + + + + +Stage + + + + +Replicate + + + + +Stage + + + + +Vote + + + + +Bank + + + + +Stage + + + + +Validators + + + diff --git a/src/fullnode.rs b/src/fullnode.rs index 0cf1ae28d4..1cde1e535f 100644 --- a/src/fullnode.rs +++ b/src/fullnode.rs @@ -185,55 +185,6 @@ impl Fullnode { } /// Create a fullnode instance acting as a leader or validator. - /// - /// ```text - /// .---------------------. - /// | Leader | - /// | | - /// .--------. | .-----. | - /// | |---->| | | - /// | Client | | | RPU | | - /// | |<----| | | - /// `----+---` | `-----` | - /// | | ^ | - /// | | | | - /// | | .--+---. | - /// | | | Bank | | - /// | | `------` | - /// | | ^ | - /// | | | | .------------. - /// | | .--+--. .-----. | | | - /// `-------->| TPU +-->| NCP +------>| Validators | - /// | `-----` `-----` | | | - /// | | `------------` - /// `---------------------` - /// - /// .-------------------------------. - /// | Validator | - /// | | - /// .--------. | .-----. | - /// | |-------------->| | | - /// | Client | | | RPU | | - /// | |<--------------| | | - /// `--------` | `-----` | - /// | ^ | - /// | | | - /// | .--+---. | - /// | | Bank | | - /// | `------` | - /// | ^ | - /// .--------. | | | .------------. - /// | | | .--+--. | | | - /// | Leader |<------------->| TVU +<--------------->| | - /// | | | `-----` | | Validators | - /// | | | ^ | | | - /// | | | | | | | - /// | | | .--+--. | | | - /// | |<------------->| NCP +<--------------->| | - /// | | | `-----` | | | - /// `--------` | | `------------` - /// `-------------------------------` - /// ``` #[cfg_attr(feature = "cargo-clippy", allow(too_many_arguments))] pub fn new_with_bank( keypair: Arc, diff --git a/src/tpu.rs b/src/tpu.rs index 1b3c680ec3..41095a87dd 100644 --- a/src/tpu.rs +++ b/src/tpu.rs @@ -1,29 +1,5 @@ //! The `tpu` module implements the Transaction Processing Unit, a //! 5-stage transaction processing pipeline in software. -//! -//! ```text -//! .----------------------------------------------------. -//! | TPU .-------------. | -//! | | PoH Service | | -//! | `-------+-----` | -//! | ^ | | -//! | | v | -//! | .-------. .-----------. .-+-------. .-------. | -//! .---------. | | Fetch | | SigVerify | | Banking | | Write | | .------------. -//! | Clients |--->| Stage |->| Stage |->| Stage |-->| Stage +--->| Validators | -//! `---------` | | | | | | | | | | `------------` -//! | `-------` `-----------` `----+----` `---+---` | -//! | | | | -//! | | | | -//! | | | | -//! | | | | -//! `---------------------------------|------------|-----` -//! | | -//! v v -//! .------. .--------. -//! | Bank | | Ledger | -//! `------` `--------` -//! ``` use bank::Bank; use banking_stage::{BankingStage, BankingStageReturnType}; diff --git a/src/tvu.rs b/src/tvu.rs index 799db33582..cc0c0c4799 100644 --- a/src/tvu.rs +++ b/src/tvu.rs @@ -1,32 +1,6 @@ //! The `tvu` module implements the Transaction Validation Unit, a //! 3-stage transaction validation pipeline in software. //! -//! ```text -//! .------------------------------------------------. -//! | | -//! | .------------------------------------+------------. -//! | | TVU | | -//! | | | | -//! | | | | .------------. -//! | | .----------------+-------------->| Validators | -//! v | .-------. | | | `------------` -//! .----+---. | | | .----+-------. .----+---------. | -//! | Leader |--------->| Blob | | Retransmit | | Replicate | | -//! `--------` | | Fetch |-->| Stage |-->| Stage / | | -//! .------------. | | Stage | | | | Vote Stage | | -//! | Validators |----->| | `------------` `----+---------` | -//! `------------` | `-------` | | -//! | | | -//! | | | -//! | | | -//! `------------------------------------|------------` -//! | -//! v -//! .------. -//! | Bank | -//! `------` -//! ``` -//! //! 1. Fetch Stage //! - Incoming blobs are picked up from the replicate socket and repair socket. //! 2. SharedWindow Stage