From b3d5ce7d48426bbe9269f3ea89029187cf939398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 10 Jun 2015 01:20:35 +0300 Subject: [PATCH 1/4] cmd/geth, eth/downloader: collect and report import progress too --- cmd/geth/admin.go | 8 ++++---- eth/downloader/downloader.go | 39 ++++++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/cmd/geth/admin.go b/cmd/geth/admin.go index 13d10de32..ea8a70923 100644 --- a/cmd/geth/admin.go +++ b/cmd/geth/admin.go @@ -51,7 +51,7 @@ func (js *jsre) adminBindings() { admin.Set("import", js.importChain) admin.Set("export", js.exportChain) admin.Set("verbosity", js.verbosity) - admin.Set("progress", js.downloadProgress) + admin.Set("progress", js.syncProgress) admin.Set("setSolc", js.setSolc) admin.Set("contractInfo", struct{}{}) @@ -324,9 +324,9 @@ func (js *jsre) setHead(call otto.FunctionCall) otto.Value { return otto.UndefinedValue() } -func (js *jsre) downloadProgress(call otto.FunctionCall) otto.Value { - pending, cached := js.ethereum.Downloader().Stats() - v, _ := call.Otto.ToValue(map[string]interface{}{"pending": pending, "cached": cached}) +func (js *jsre) syncProgress(call otto.FunctionCall) otto.Value { + pending, cached, importing := js.ethereum.Downloader().Stats() + v, _ := call.Otto.ToValue(map[string]interface{}{"pending": pending, "cached": cached, "importing": importing}) return v } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 29b627771..efb94e5e3 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -78,6 +78,10 @@ type Downloader struct { checks map[common.Hash]*crossCheck // Pending cross checks to verify a hash chain banned *set.Set // Set of hashes we've received and banned + // Statistics + importQueue []common.Hash // Hashes of the previously taken blocks to check import progress + importLock sync.Mutex + // Callbacks hasBlock hashCheckFn getBlock getBlockFn @@ -121,8 +125,21 @@ func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock getBlockFn) *Downloa return downloader } -func (d *Downloader) Stats() (current int, max int) { - return d.queue.Size() +// Stats retrieves the current status of the downloader. +func (d *Downloader) Stats() (pending int, cached int, importing int) { + // Fetch the download status + pending, cached = d.queue.Size() + + // Generate the import status + d.importLock.Lock() + defer d.importLock.Unlock() + + for len(d.importQueue) > 0 && d.hasBlock(d.importQueue[0]) { + d.importQueue = d.importQueue[1:] + } + importing = len(d.importQueue) + + return } // Synchronising returns the state of the downloader @@ -202,7 +219,17 @@ func (d *Downloader) Synchronise(id string, hash common.Hash) error { // TakeBlocks takes blocks from the queue and yields them to the caller. func (d *Downloader) TakeBlocks() []*Block { - return d.queue.TakeBlocks() + blocks := d.queue.TakeBlocks() + if len(blocks) > 0 { + hashes := make([]common.Hash, len(blocks)) + for i, block := range blocks { + hashes[i] = block.RawBlock.Hash() + } + d.importLock.Lock() + d.importQueue = hashes + d.importLock.Unlock() + } + return blocks } // Has checks if the downloader knows about a particular hash, meaning that its @@ -255,9 +282,13 @@ func (d *Downloader) Cancel() bool { } d.cancelLock.Unlock() - // reset the queue + // Reset the queue and import statistics d.queue.Reset() + d.importLock.Lock() + d.importQueue = nil + d.importLock.Unlock() + return true } From 271fb20ecb48944692bdb4e7f91be9b90506981b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 10 Jun 2015 18:01:05 +0300 Subject: [PATCH 2/4] cmd/geth, eth/downloader: rough guess at the import eta --- cmd/geth/admin.go | 9 +++++++-- eth/downloader/downloader.go | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/geth/admin.go b/cmd/geth/admin.go index ea8a70923..7bf23829a 100644 --- a/cmd/geth/admin.go +++ b/cmd/geth/admin.go @@ -325,8 +325,13 @@ func (js *jsre) setHead(call otto.FunctionCall) otto.Value { } func (js *jsre) syncProgress(call otto.FunctionCall) otto.Value { - pending, cached, importing := js.ethereum.Downloader().Stats() - v, _ := call.Otto.ToValue(map[string]interface{}{"pending": pending, "cached": cached, "importing": importing}) + pending, cached, importing, eta := js.ethereum.Downloader().Stats() + v, _ := call.Otto.ToValue(map[string]interface{}{ + "pending": pending, + "cached": cached, + "importing": importing, + "estimate": eta.String(), + }) return v } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index efb94e5e3..c3234ecb1 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -79,7 +79,9 @@ type Downloader struct { banned *set.Set // Set of hashes we've received and banned // Statistics + importStart time.Time // Instance when the last blocks were taken from the cache importQueue []common.Hash // Hashes of the previously taken blocks to check import progress + importDone int // Number of taken blocks already imported from the last batch importLock sync.Mutex // Callbacks @@ -126,19 +128,25 @@ func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock getBlockFn) *Downloa } // Stats retrieves the current status of the downloader. -func (d *Downloader) Stats() (pending int, cached int, importing int) { +func (d *Downloader) Stats() (pending int, cached int, importing int, estimate time.Duration) { // Fetch the download status pending, cached = d.queue.Size() - // Generate the import status + // Figure out the import progress d.importLock.Lock() defer d.importLock.Unlock() for len(d.importQueue) > 0 && d.hasBlock(d.importQueue[0]) { d.importQueue = d.importQueue[1:] + d.importDone++ } importing = len(d.importQueue) + // Make an estimate on the total sync + estimate = 0 + if d.importDone > 0 { + estimate = time.Since(d.importStart) / time.Duration(d.importDone) * time.Duration(pending+cached+importing) + } return } @@ -226,7 +234,9 @@ func (d *Downloader) TakeBlocks() []*Block { hashes[i] = block.RawBlock.Hash() } d.importLock.Lock() + d.importStart = time.Now() d.importQueue = hashes + d.importDone = 0 d.importLock.Unlock() } return blocks @@ -287,6 +297,7 @@ func (d *Downloader) Cancel() bool { d.importLock.Lock() d.importQueue = nil + d.importDone = 0 d.importLock.Unlock() return true From c4af70d0cc213b5fd906a385cdb2895268525ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 10 Jun 2015 18:07:23 +0300 Subject: [PATCH 3/4] cmd/geth: round the import ETA before converting to string --- cmd/geth/admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/admin.go b/cmd/geth/admin.go index 7bf23829a..4f22110ad 100644 --- a/cmd/geth/admin.go +++ b/cmd/geth/admin.go @@ -330,7 +330,7 @@ func (js *jsre) syncProgress(call otto.FunctionCall) otto.Value { "pending": pending, "cached": cached, "importing": importing, - "estimate": eta.String(), + "estimate": (eta / time.Second * time.Second).String(), }) return v } From 3c1cccc801ec7c546a0d840fe3a08dbf9a302d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 10 Jun 2015 20:12:22 +0300 Subject: [PATCH 4/4] eth/downloader: fetch the block hashes on the fly, when needed --- eth/downloader/downloader.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index c3234ecb1..f0a515d12 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -79,9 +79,9 @@ type Downloader struct { banned *set.Set // Set of hashes we've received and banned // Statistics - importStart time.Time // Instance when the last blocks were taken from the cache - importQueue []common.Hash // Hashes of the previously taken blocks to check import progress - importDone int // Number of taken blocks already imported from the last batch + importStart time.Time // Instance when the last blocks were taken from the cache + importQueue []*Block // Previously taken blocks to check import progress + importDone int // Number of taken blocks already imported from the last batch importLock sync.Mutex // Callbacks @@ -136,7 +136,7 @@ func (d *Downloader) Stats() (pending int, cached int, importing int, estimate t d.importLock.Lock() defer d.importLock.Unlock() - for len(d.importQueue) > 0 && d.hasBlock(d.importQueue[0]) { + for len(d.importQueue) > 0 && d.hasBlock(d.importQueue[0].RawBlock.Hash()) { d.importQueue = d.importQueue[1:] d.importDone++ } @@ -229,13 +229,9 @@ func (d *Downloader) Synchronise(id string, hash common.Hash) error { func (d *Downloader) TakeBlocks() []*Block { blocks := d.queue.TakeBlocks() if len(blocks) > 0 { - hashes := make([]common.Hash, len(blocks)) - for i, block := range blocks { - hashes[i] = block.RawBlock.Hash() - } d.importLock.Lock() d.importStart = time.Now() - d.importQueue = hashes + d.importQueue = blocks d.importDone = 0 d.importLock.Unlock() }