Merge pull request #1199 from LukasKorba/1181-Correct-computation-of-progress-for-Spend-before-Sync

[#1181] Correct computation of progress for Spend before Sync

CHANGELOG.md

@@ -8,6 +8,13 @@ be able to write to this location after it creates this directory. It is suggest
 a subdirectory of the `Documents` directory. If this information is stored in `Documents` then the
 system itself won't remove these data.
 
+### Removed
+
+### [#1181] Correct computation of progress for Spend before Sync
+`latestScannedHeight` and `latestScannedTime` have been removed from `SynchronizerState`. With multiple algorithms
+of syncing the amount of data provided is reduced so it's consistent. Spend before Sync is done in non-linear order
+so both Height and Time don't make sense anymore. 
+
 # 0.22.0-beta
 
 ## Checkpoints

Example/ZcashLightClientSample/ZcashLightClientSample/DemoAppConfig.swift

@@ -18,7 +18,7 @@ enum DemoAppConfig {
         let seed: [UInt8]
     }
 
-    static let host = ZcashSDK.isMainnet ? "mainnet.lightwalletd.com" : "testnet.lightwalletd.com"
+    static let host = ZcashSDK.isMainnet ? "mainnet.lightwalletd.com" : "lightwalletd.testnet.electriccoin.co"
     static let port: Int = 9067
 
 //    static let defaultBirthdayHeight: BlockHeight = ZcashSDK.isMainnet ? 935000 : 1386000

Example/ZcashLightClientSample/ZcashLightClientSample/Sync Blocks/SyncBlocksListViewController.swift

@@ -112,7 +112,6 @@ class SyncBlocksListViewController: UIViewController {
             outputParamsURL: try! outputParamsURLHelper(),
             saplingParamsSourceURL: SaplingParamsSourceURL.default,
             alias: data.alias,
-            syncAlgorithm: .spendBeforeSync,
             loggingPolicy: .default(.debug),
             enableBackendTracing: true
         )

Example/ZcashLightClientSample/ZcashLightClientSample/Sync Blocks/SyncBlocksViewController.swift

@@ -84,10 +84,7 @@ class SyncBlocksViewController: UIViewController {
 
             progressBar.progress = progress
             progressLabel.text = "\(floor(progress * 1000) / 10)%"
-            let syncedDate = dateFormatter.string(from: Date(timeIntervalSince1970: state.latestScannedTime))
             let progressText = """
-            synced date         \(syncedDate)
-            synced block        \(state.latestScannedHeight)
             latest block height \(state.latestBlockHeight)
             """
             progressDataLabel.text = progressText

Sources/ZcashLightClientKit/Block/Actions/Action.swift

@@ -14,7 +14,10 @@ actor ActionContext {
     let preferredSyncAlgorithm: SyncAlgorithm
     var supportedSyncAlgorithm: SyncAlgorithm?
     var requestedRewindHeight: BlockHeight?
+    /// Represents the overall range of blocks that will be synced, `SyncAlgorithm` doesn't matter.
     var totalProgressRange: CompactBlockRange = 0...0
+    /// Amount of blocks that have been processed so far
+    var processedHeight: BlockHeight = 0
     var lastScannedHeight: BlockHeight?
     var lastDownloadedHeight: BlockHeight?
     var lastEnhancedHeight: BlockHeight?
@@ -30,7 +33,11 @@ actor ActionContext {
         self.state = state
     }
     func update(syncControlData: SyncControlData) async { self.syncControlData = syncControlData }
-    func update(totalProgressRange: CompactBlockRange) async { self.totalProgressRange = totalProgressRange }
+    func update(totalProgressRange: CompactBlockRange) async {
+        self.processedHeight = totalProgressRange.lowerBound
+        self.totalProgressRange = totalProgressRange
+    }
+    func update(processedHeight: BlockHeight) async { self.processedHeight = processedHeight }
     func update(lastScannedHeight: BlockHeight) async { self.lastScannedHeight = lastScannedHeight }
     func update(lastDownloadedHeight: BlockHeight) async { self.lastDownloadedHeight = lastDownloadedHeight }
     func update(lastEnhancedHeight: BlockHeight?) async { self.lastEnhancedHeight = lastEnhancedHeight }

Sources/ZcashLightClientKit/Block/Actions/ProcessSuggestedScanRangesAction.swift

@@ -46,7 +46,21 @@ extension ProcessSuggestedScanRangesAction: Action {
             await context.update(lastScannedHeight: lowerBound)
             await context.update(lastDownloadedHeight: lowerBound)
             await context.update(syncControlData: syncControlData)
-            await context.update(totalProgressRange: lowerBound...upperBound)
+            
+            // the total progress range is computed only for the first time
+            // as a sum of all ranges
+            let totalProgressRange = await context.totalProgressRange
+            if totalProgressRange.lowerBound == 0 && totalProgressRange.upperBound == 0 {
+                var minHeight = Int.max
+                var maxHeight = 0
+                scanRanges.forEach { range in
+                    if range.range.lowerBound < minHeight { minHeight = range.range.lowerBound }
+                    if range.range.upperBound > maxHeight { maxHeight = range.range.upperBound }
+                }
+                
+                logger.info("Setting the total range for Spend before Sync to \(minHeight...maxHeight).")
+                await context.update(totalProgressRange: minHeight...maxHeight)
+            }
             
             // If there is a range of blocks that needs to be verified, it will always
             // be returned as the first element of the vector of suggested ranges.

Sources/ZcashLightClientKit/Block/Actions/ScanAction.swift

@@ -51,11 +51,20 @@ extension ScanAction: Action {
         let totalProgressRange = await context.totalProgressRange
         
         do {
-            try await blockScanner.scanBlocks(at: batchRange, totalProgressRange: totalProgressRange) { [weak self] lastScannedHeight in
+            try await blockScanner.scanBlocks(at: batchRange, totalProgressRange: totalProgressRange) { [weak self] lastScannedHeight, increment in
+                let processedHeight = await context.processedHeight
+                let incrementedprocessedHeight = processedHeight + BlockHeight(increment)
+                await context.update(
+                    processedHeight:
+                        incrementedprocessedHeight < totalProgressRange.upperBound
+                        ? incrementedprocessedHeight
+                        : totalProgressRange.upperBound
+                )
+
                 let progress = BlockProgress(
                     startHeight: totalProgressRange.lowerBound,
                     targetHeight: totalProgressRange.upperBound,
-                    progressHeight: lastScannedHeight
+                    progressHeight: incrementedprocessedHeight
                 )
                 self?.logger.debug("progress: \(progress)")
                 await didUpdate(.progressPartialUpdate(.syncing(progress)))

Sources/ZcashLightClientKit/Block/CompactBlockProcessor.swift

@@ -521,7 +521,7 @@ extension CompactBlockProcessor {
                 // Execute action.
                 context = try await action.run(with: context) { [weak self] event in
                     await self?.send(event: event)
-                    if let progressChanged = await self?.compactBlockProgress.event(event), progressChanged {
+                    if let progressChanged = await self?.compactBlockProgress.hasProgressUpdated(event), progressChanged {
                         if let progress = await self?.compactBlockProgress.progress {
                             await self?.send(event: .progressUpdated(progress))
                         }

Sources/ZcashLightClientKit/Block/Scan/BlockScanner.swift

@@ -17,7 +17,7 @@ protocol BlockScanner {
     func scanBlocks(
         at range: CompactBlockRange,
         totalProgressRange: CompactBlockRange,
-        didScan: @escaping (BlockHeight) async -> Void
+        didScan: @escaping (BlockHeight, UInt32) async -> Void
     ) async throws -> BlockHeight
 }
 
@@ -35,7 +35,7 @@ extension BlockScannerImpl: BlockScanner {
     func scanBlocks(
         at range: CompactBlockRange,
         totalProgressRange: CompactBlockRange,
-        didScan: @escaping (BlockHeight) async -> Void
+        didScan: @escaping (BlockHeight, UInt32) async -> Void
     ) async throws -> BlockHeight {
         logger.debug("Going to scan blocks in range: \(range)")
         try Task.checkCancellation()
@@ -69,7 +69,7 @@ extension BlockScannerImpl: BlockScanner {
             
             scannedNewBlocks = previousScannedHeight != lastScannedHeight
             if scannedNewBlocks {
-                await didScan(lastScannedHeight)
+                await didScan(lastScannedHeight, batchSize)
 
                 let progress = BlockProgress(
                     startHeight: totalProgressRange.lowerBound,

Sources/ZcashLightClientKit/Block/Utils/CompactBlockProgress.swift

@@ -19,7 +19,7 @@ final actor CompactBlockProgress {
             switch self {
             case .enhance: return 0.08
             case .fetch: return 0.02
-            case .scan: return 0.9
+            case .scan: return 1.0
             }
         }
     }
@@ -35,18 +35,13 @@ final actor CompactBlockProgress {
         return overallProgress
     }
     
-    func event(_ event: CompactBlockProcessor.Event) -> Bool {
+    func hasProgressUpdated(_ event: CompactBlockProcessor.Event) -> Bool {
         guard case .progressPartialUpdate(let update) = event else {
             return false
         }
         
-        switch update {
+        if case let .syncing(progress) = update {
-        case .syncing(let progress):
             actionProgresses[.scan] = progress.progress
-        case .enhance(let progress):
-            actionProgresses[.enhance] = progress.progress
-        case .fetch(let progress):
-            actionProgresses[.fetch] = progress
         }
         
         return true

Sources/ZcashLightClientKit/Synchronizer.swift

@@ -42,13 +42,8 @@ public struct SynchronizerState: Equatable {
     /// status of the whole sync process
     var internalSyncStatus: InternalSyncStatus
     public var syncStatus: SyncStatus
-    /// height of the latest scanned block known to this synchronizer.
-    public var latestScannedHeight: BlockHeight
     /// height of the latest block on the blockchain known to this synchronizer.
     public var latestBlockHeight: BlockHeight
-    /// timestamp of the latest scanned block on the blockchain known to this synchronizer.
-    /// The anchor point is timeIntervalSince1970
-    public var latestScannedTime: TimeInterval
 
     /// Represents a synchronizer that has made zero progress hasn't done a sync attempt
     public static var zero: SynchronizerState {
@@ -76,9 +71,7 @@ public struct SynchronizerState: Equatable {
         self.shieldedBalance = shieldedBalance
         self.transparentBalance = transparentBalance
         self.internalSyncStatus = internalSyncStatus
-        self.latestScannedHeight = latestScannedHeight
         self.latestBlockHeight = latestBlockHeight
-        self.latestScannedTime = latestScannedTime
         self.syncStatus = internalSyncStatus.mapToSyncStatus()
     }
 }

Sources/ZcashLightClientKit/Synchronizer/SDKSynchronizer.swift

@@ -369,7 +369,7 @@ public class SDKSynchronizer: Synchronizer {
     }
 
     public func allPendingTransactions() async throws -> [ZcashTransaction.Overview] {
-        let latestScannedHeight = self.latestState.latestScannedHeight
+        let latestScannedHeight = try await transactionRepository.lastScannedHeight()
         
         return try await transactionRepository.findPendingTransactions(latestHeight: latestScannedHeight, offset: 0, limit: .max)
     }

Tests/TestUtils/Sourcery/GeneratedMocks/AutoMockable.generated.swift

@@ -361,11 +361,11 @@ class BlockScannerMock: BlockScanner {
     var scanBlocksAtTotalProgressRangeDidScanCalled: Bool {
         return scanBlocksAtTotalProgressRangeDidScanCallsCount > 0
     }
-    var scanBlocksAtTotalProgressRangeDidScanReceivedArguments: (range: CompactBlockRange, totalProgressRange: CompactBlockRange, didScan: (BlockHeight) async -> Void)?
+    var scanBlocksAtTotalProgressRangeDidScanReceivedArguments: (range: CompactBlockRange, totalProgressRange: CompactBlockRange, didScan: (BlockHeight, UInt32) async -> Void)?
     var scanBlocksAtTotalProgressRangeDidScanReturnValue: BlockHeight!
-    var scanBlocksAtTotalProgressRangeDidScanClosure: ((CompactBlockRange, CompactBlockRange, @escaping (BlockHeight) async -> Void) async throws -> BlockHeight)?
+    var scanBlocksAtTotalProgressRangeDidScanClosure: ((CompactBlockRange, CompactBlockRange, @escaping (BlockHeight, UInt32) async -> Void) async throws -> BlockHeight)?
 
-    func scanBlocks(at range: CompactBlockRange, totalProgressRange: CompactBlockRange, didScan: @escaping (BlockHeight) async -> Void) async throws -> BlockHeight {
+    func scanBlocks(at range: CompactBlockRange, totalProgressRange: CompactBlockRange, didScan: @escaping (BlockHeight, UInt32) async -> Void) async throws -> BlockHeight {
         if let error = scanBlocksAtTotalProgressRangeDidScanThrowableError {
             throw error
         }