From 2924fdfcf7ab67a66a4ed3fb95cb9140be0cc809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 6 Sep 2016 12:39:14 +0300 Subject: [PATCH] ethereum, ethclient: add SyncProgress API endpoint --- eth/downloader/api.go | 21 ++++------ eth/downloader/downloader.go | 11 +++++- eth/downloader/downloader_test.go | 64 +++++++++++++++---------------- ethclient/ethclient.go | 33 ++++++++++++++++ ethclient/ethclient_test.go | 1 + interfaces.go | 16 ++++++++ internal/ethapi/api.go | 14 +++---- 7 files changed, 105 insertions(+), 55 deletions(-) diff --git a/eth/downloader/api.go b/eth/downloader/api.go index c36dfb7e0..e41376810 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -19,6 +19,7 @@ package downloader import ( "sync" + ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" "golang.org/x/net/context" @@ -73,9 +74,10 @@ func (api *PublicDownloaderAPI) eventLoop() { var notification interface{} switch event.Data.(type) { case StartEvent: - result := &SyncingResult{Syncing: true} - result.Status.Origin, result.Status.Current, result.Status.Height, result.Status.Pulled, result.Status.Known = api.d.Progress() - notification = result + notification = &SyncingResult{ + Syncing: true, + Status: api.d.Progress(), + } case DoneEvent, FailedEvent: notification = false } @@ -117,19 +119,10 @@ func (api *PublicDownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, return rpcSub, nil } -// Progress gives progress indications when the node is synchronising with the Ethereum network. -type Progress struct { - Origin uint64 `json:"startingBlock"` - Current uint64 `json:"currentBlock"` - Height uint64 `json:"highestBlock"` - Pulled uint64 `json:"pulledStates"` - Known uint64 `json:"knownStates"` -} - // SyncingResult provides information about the current synchronisation status for this node. type SyncingResult struct { - Syncing bool `json:"syncing"` - Status Progress `json:"status"` + Syncing bool `json:"syncing"` + Status ethereum.SyncProgress `json:"status"` } // uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index c8f710450..fb63757aa 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -28,6 +28,7 @@ import ( "sync/atomic" "time" + ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -211,7 +212,7 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, ha // In addition, during the state download phase of fast synchronisation the number // of processed and the total number of known states are also returned. Otherwise // these are zero. -func (d *Downloader) Progress() (uint64, uint64, uint64, uint64, uint64) { +func (d *Downloader) Progress() ethereum.SyncProgress { // Fetch the pending state count outside of the lock to prevent unforeseen deadlocks pendingStates := uint64(d.queue.PendingNodeData()) @@ -228,7 +229,13 @@ func (d *Downloader) Progress() (uint64, uint64, uint64, uint64, uint64) { case LightSync: current = d.headHeader().Number.Uint64() } - return d.syncStatsChainOrigin, current, d.syncStatsChainHeight, d.syncStatsStateDone, d.syncStatsStateDone + pendingStates + return ethereum.SyncProgress{ + StartingBlock: d.syncStatsChainOrigin, + CurrentBlock: current, + HighestBlock: d.syncStatsChainHeight, + PulledStates: d.syncStatsStateDone, + KnownStates: d.syncStatsStateDone + pendingStates, + } } // Synchronising returns whether the downloader is currently retrieving blocks. diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a2efc7469..ae9a85ae1 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -1377,8 +1377,8 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { <-progress } // Retrieve the sync progress and ensure they are zero (pristine sync) - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) } // Synchronise half the blocks and check initial progress tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts) @@ -1392,8 +1392,8 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks/2+1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks/2+1) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1) } progress <- struct{}{} pending.Wait() @@ -1409,15 +1409,15 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks) } progress <- struct{}{} pending.Wait() // Check final progress after successful sync - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks) } } @@ -1450,8 +1450,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { <-progress } // Retrieve the sync progress and ensure they are zero (pristine sync) - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) } // Synchronise with one of the forks and check progress tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA) @@ -1465,8 +1465,8 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(len(hashesA)-1) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, len(hashesA)-1) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(len(hashesA)-1) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, len(hashesA)-1) } progress <- struct{}{} pending.Wait() @@ -1485,15 +1485,15 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesA)-1) || latest != uint64(len(hashesB)-1) { - t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesA)-1, len(hashesB)-1) + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesA)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { + t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesA)-1, len(hashesB)-1) } progress <- struct{}{} pending.Wait() // Check final progress after successful sync - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesB)-1) || latest != uint64(len(hashesB)-1) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesB)-1, len(hashesB)-1) + if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(common) || progress.CurrentBlock != uint64(len(hashesB)-1) || progress.HighestBlock != uint64(len(hashesB)-1) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, common, len(hashesB)-1, len(hashesB)-1) } } @@ -1526,8 +1526,8 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { <-progress } // Retrieve the sync progress and ensure they are zero (pristine sync) - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) } // Attempt a full sync with a faulty peer tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts) @@ -1546,8 +1546,8 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks) } progress <- struct{}{} pending.Wait() @@ -1563,15 +1563,15 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks/2) || latest != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks/2, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks/2) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks/2, targetBlocks) } progress <- struct{}{} pending.Wait() // Check final progress after successful sync - if origin, current, latest, _, _ := tester.downloader.Progress(); origin > uint64(targetBlocks/2) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks/2, targetBlocks, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks/2) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2, targetBlocks, targetBlocks) } } @@ -1603,8 +1603,8 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { <-progress } // Retrieve the sync progress and ensure they are zero (pristine sync) - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 { - t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 { + t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0) } // Create and sync with an attacker that promises a higher chain than available tester.newPeer("attack", protocol, hashes, headers, blocks, receipts) @@ -1624,8 +1624,8 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks+3) { - t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks+3) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks+3) { + t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks+3) } progress <- struct{}{} pending.Wait() @@ -1641,15 +1641,15 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) { } }() <-starting - if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks) || latest != uint64(targetBlocks) { - t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock > uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, targetBlocks, targetBlocks) } progress <- struct{}{} pending.Wait() // Check final progress after successful sync - if origin, current, latest, _, _ := tester.downloader.Progress(); origin > uint64(targetBlocks) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) { - t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks, targetBlocks, targetBlocks) + if progress := tester.downloader.Progress(); progress.StartingBlock > uint64(targetBlocks) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) { + t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks, targetBlocks, targetBlocks) } } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 575a7cb44..50346f6ae 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -191,6 +191,39 @@ func toBlockNumArg(number *big.Int) string { return fmt.Sprintf("%#x", number) } +type rpcProgress struct { + StartingBlock rpc.HexNumber + CurrentBlock rpc.HexNumber + HighestBlock rpc.HexNumber + PulledStates rpc.HexNumber + KnownStates rpc.HexNumber +} + +// SyncProgress retrieves the current progress of the sync algorithm. If there's +// no sync currently running, it returns nil. +func (ec *Client) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, error) { + var raw json.RawMessage + if err := ec.c.CallContext(ctx, &raw, "eth_syncing"); err != nil { + return nil, err + } + // Handle the possible response types + var syncing bool + if err := json.Unmarshal(raw, &syncing); err == nil { + return nil, nil // Not syncing (always false) + } + var progress *rpcProgress + if err := json.Unmarshal(raw, &progress); err != nil { + return nil, err + } + return ðereum.SyncProgress{ + StartingBlock: progress.StartingBlock.Uint64(), + CurrentBlock: progress.CurrentBlock.Uint64(), + HighestBlock: progress.HighestBlock.Uint64(), + PulledStates: progress.PulledStates.Uint64(), + KnownStates: progress.KnownStates.Uint64(), + }, nil +} + // SubscribeNewHead subscribes to notifications about the current blockchain head // on the given channel. func (ec *Client) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 47e37c0ce..0cc11eb5b 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -6,6 +6,7 @@ import "github.com/ethereum/go-ethereum" var ( _ = ethereum.ChainReader(&Client{}) _ = ethereum.ChainStateReader(&Client{}) + _ = ethereum.ChainSyncReader(&Client{}) _ = ethereum.ChainHeadEventer(&Client{}) _ = ethereum.ContractCaller(&Client{}) _ = ethereum.GasEstimator(&Client{}) diff --git a/interfaces.go b/interfaces.go index 921d02616..aab0e2029 100644 --- a/interfaces.go +++ b/interfaces.go @@ -67,6 +67,22 @@ type ChainStateReader interface { NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) } +// SyncProgress gives progress indications when the node is synchronising with +// the Ethereum network. +type SyncProgress struct { + StartingBlock uint64 // Block number where sync began + CurrentBlock uint64 // Current block number where sync is at + HighestBlock uint64 // Highest alleged block number in the chain + PulledStates uint64 // Number of state trie entries already downloaded + KnownStates uint64 // Total number os state trie entries known about +} + +// ChainSyncReader wraps access to the node's current sync status. If there's no +// sync currently running, it returns nil. +type ChainSyncReader interface { + SyncProgress(ctx context.Context) (*SyncProgress, error) +} + // A ChainHeadEventer returns notifications whenever the canonical head block is updated. type ChainHeadEventer interface { SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (Subscription, error) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0fba55ae8..0b1384f58 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -73,19 +73,19 @@ func (s *PublicEthereumAPI) ProtocolVersion() *rpc.HexNumber { // - pulledStates: number of state entries processed until now // - knownStates: number of known state entries that still need to be pulled func (s *PublicEthereumAPI) Syncing() (interface{}, error) { - origin, current, height, pulled, known := s.b.Downloader().Progress() + progress := s.b.Downloader().Progress() // Return not syncing if the synchronisation already completed - if current >= height { + if progress.CurrentBlock >= progress.HighestBlock { return false, nil } // Otherwise gather the block sync stats return map[string]interface{}{ - "startingBlock": rpc.NewHexNumber(origin), - "currentBlock": rpc.NewHexNumber(current), - "highestBlock": rpc.NewHexNumber(height), - "pulledStates": rpc.NewHexNumber(pulled), - "knownStates": rpc.NewHexNumber(known), + "startingBlock": rpc.NewHexNumber(progress.StartingBlock), + "currentBlock": rpc.NewHexNumber(progress.CurrentBlock), + "highestBlock": rpc.NewHexNumber(progress.HighestBlock), + "pulledStates": rpc.NewHexNumber(progress.PulledStates), + "knownStates": rpc.NewHexNumber(progress.KnownStates), }, nil }