From dfaa827fd14ea246c378398b9ac201897d7f18a0 Mon Sep 17 00:00:00 2001 From: Honza Rychnovsky Date: Wed, 10 May 2023 12:45:23 +0200 Subject: [PATCH] [#1013] Sync stages parallelization * [#1013] Sync blockchain sub-phases parallelization - Remove the unnecessary comment after the latest changes in this code fragment * Move solely CompactBlockProcessor-related constants * Simplify sync range update construction * [#1013] Sync blockchain sub-phases parallelization * Changelog update * Block files deletion documentation update * Leverage buildList API * CompactBlockRepository documentation update * Move BlockBatch to internal models --- CHANGELOG.md | 12 + .../demos/getbalance/GetBalanceFragment.kt | 7 +- .../ListTransactionsFragment.kt | 5 +- .../demos/listutxos/ListUtxosFragment.kt | 5 +- .../sdk/demoapp/demos/send/SendFragment.kt | 7 +- .../screen/home/viewmodel/WalletViewModel.kt | 8 +- .../android/sdk/model/PercentDecimalTest.kt | 19 - .../block/FileCompactBlockRepositoryTest.kt | 18 +- .../android/sdk/model/PercentDecimalTest.kt | 38 ++ .../cash/z/ecc/android/sdk/SdkSynchronizer.kt | 19 +- .../cash/z/ecc/android/sdk/Synchronizer.kt | 8 +- .../sdk/block/CompactBlockProcessor.kt | 333 +++++++++++------- .../cash/z/ecc/android/sdk/ext/ZcashSdk.kt | 23 -- .../internal/block/CompactBlockDownloader.kt | 6 +- .../android/sdk/internal/model/BlockBatch.kt | 9 + .../repository/CompactBlockRepository.kt | 26 +- .../block/FileCompactBlockRepository.kt | 48 ++- .../z/ecc/android/sdk/model/PercentDecimal.kt | 11 +- 18 files changed, 358 insertions(+), 244 deletions(-) delete mode 100644 sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt create mode 100644 sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt create mode 100644 sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt rename {sdk-incubator-lib => sdk-lib}/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt (67%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c7abc58..948b65ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ Change Log ========== +## Unreleased +- The SDK's `CompactBlockProcessor` switched from processing **all blocks in one run** mechanism to **batched blocks** +processing. This was necessary for the sync state's parallelization. Example of syncing of the latest + 100 blocks: + - Previously: _Download 100 blocks -> Validate 100 blocks -> Scan 100 blocks -> SYNCED_ + - Now: _10x (Download 10 blocks -> Validate 10 blocks -> Scan 10 blocks) -> SYNCED_ +- `Synchronizer.progress` now returns `Flow` instead of `Flow`. PercentDecimal is a type-safe + model. + Use `PercentDecimal.toPercentage()` to get a number within 0-100% scale. +- `Synchronizer.status` now provides a new `SYNCING` state, which covers all three previous `DOWNLOADING`, + `VALIDATING`, and `SCANNING` states, which were eliminated in favor of `SYNCING` state. + ## 1.17.0-beta01 - Synchronizer APIs for listing sent and received transactions have been removed. - Synchronizer APIs for listing pending transactions have been removed, along with the `PendingTransaction` object. diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt index 582b0333..6a27887e 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/getbalance/GetBalanceFragment.kt @@ -21,6 +21,7 @@ import cash.z.ecc.android.sdk.ext.ZcashSdk import cash.z.ecc.android.sdk.ext.convertZatoshiToZecString import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.model.Account +import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.Zatoshi import cash.z.ecc.android.sdk.model.ZcashNetwork @@ -182,9 +183,9 @@ class GetBalanceFragment : BaseDemoFragment() { } @Suppress("MagicNumber") - private fun onProgress(i: Int) { - if (i < 100) { - binding.textStatus.text = "Syncing blocks...$i%" + private fun onProgress(percent: PercentDecimal) { + if (percent.isLessThanHundredPercent()) { + binding.textStatus.text = "Syncing blocks...${percent.toPercentage()}%" } } diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt index be3947c8..a00832af 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/listtransactions/ListTransactionsFragment.kt @@ -14,6 +14,7 @@ import cash.z.ecc.android.sdk.demoapp.BaseDemoFragment import cash.z.ecc.android.sdk.demoapp.R import cash.z.ecc.android.sdk.demoapp.databinding.FragmentListTransactionsBinding import cash.z.ecc.android.sdk.internal.Twig +import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.TransactionOverview import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.filterNotNull @@ -81,8 +82,8 @@ class ListTransactionsFragment : BaseDemoFragment() { } @Suppress("MagicNumber") - private fun onProgress(i: Int) { - if (i < 100) binding.textStatus.text = "Syncing blocks...$i%" + private fun onProgress(percent: PercentDecimal) { + if (percent.isLessThanHundredPercent()) binding.textStatus.text = "Syncing blocks...${percent.toPercentage()}%" } private fun onStatus(status: Synchronizer.Status) { diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt index ce07173d..b5a60b8f 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/demos/send/SendFragment.kt @@ -19,6 +19,7 @@ import cash.z.ecc.android.sdk.demoapp.util.mainActivity import cash.z.ecc.android.sdk.ext.convertZatoshiToZecString import cash.z.ecc.android.sdk.ext.convertZecToZatoshi import cash.z.ecc.android.sdk.ext.toZecString +import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -125,9 +126,9 @@ class SendFragment : BaseDemoFragment() { } @Suppress("MagicNumber") - private fun onProgress(i: Int) { - if (i < 100) { - binding.textStatus.text = "Syncing blocks...$i%" + private fun onProgress(percent: PercentDecimal) { + if (percent.isLessThanHundredPercent()) { + binding.textStatus.text = "Syncing blocks...${percent.toPercentage()}%" binding.textBalance.visibility = View.INVISIBLE } else { binding.textBalance.visibility = View.VISIBLE diff --git a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt index c4dc267f..40a3d7aa 100644 --- a/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt +++ b/demo-app/src/main/java/cash/z/ecc/android/sdk/demoapp/ui/screen/home/viewmodel/WalletViewModel.kt @@ -337,13 +337,7 @@ private fun Synchronizer.toWalletSnapshot() = val orchardBalance = flows[2] as WalletBalance? val saplingBalance = flows[3] as WalletBalance? val transparentBalance = flows[4] as WalletBalance? - - val progressPercentDecimal = (flows[5] as Int).let { value -> - if (value > PercentDecimal.MAX || value < PercentDecimal.MIN) { - PercentDecimal.ZERO_PERCENT - } - PercentDecimal(value / 100f) - } + val progressPercentDecimal = (flows[5] as PercentDecimal) WalletSnapshot( flows[0] as Synchronizer.Status, diff --git a/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt b/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt deleted file mode 100644 index d4c8bb7c..00000000 --- a/sdk-incubator-lib/src/androidTest/kotlin/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt +++ /dev/null @@ -1,19 +0,0 @@ -package cash.z.ecc.android.sdk.model - -import androidx.test.filters.SmallTest -import org.junit.Test - -class PercentDecimalTest { - - @Test(expected = IllegalArgumentException::class) - @SmallTest - fun require_greater_than_zero() { - PercentDecimal(-1.0f) - } - - @Test(expected = IllegalArgumentException::class) - @SmallTest - fun require_less_than_one() { - PercentDecimal(1.5f) - } -} diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepositoryTest.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepositoryTest.kt index f015ece7..adb30456 100644 --- a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepositoryTest.kt +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepositoryTest.kt @@ -108,9 +108,9 @@ class FileCompactBlockRepositoryTest { assertTrue { rustBackend.metadata.isEmpty() } val blocks = ListOfCompactBlocksFixture.newFlow() - val persistedCount = blockRepository.write(blocks) + val persistedBlocks = blockRepository.write(blocks) - assertEquals(blocks.count(), persistedCount) + assertEquals(blocks.count(), persistedBlocks.size) assertEquals(blocks.count(), rustBackend.metadata.size) } @@ -128,9 +128,9 @@ class FileCompactBlockRepositoryTest { assertTrue { reduced.count() < FileCompactBlockRepository.BLOCKS_METADATA_BUFFER_SIZE } } - val persistedCount = blockRepository.write(reducedBlocksList) + val persistedBlocks = blockRepository.write(reducedBlocksList) - assertEquals(reducedBlocksList.count(), persistedCount) + assertEquals(reducedBlocksList.count(), persistedBlocks.size) assertEquals(reducedBlocksList.count(), rustBackend.metadata.size) } @@ -146,10 +146,10 @@ class FileCompactBlockRepositoryTest { val blocks = ListOfCompactBlocksFixture.newFlow() - val persistedCount = blockRepository.write(blocks) + val persistedBlocks = blockRepository.write(blocks) assertTrue { rootBlocksDirectory.exists() } - assertEquals(blocks.count(), persistedCount) + assertEquals(blocks.count(), persistedBlocks.size) assertEquals(blocks.count(), rootBlocksDirectory.list()!!.size) } @@ -165,7 +165,7 @@ class FileCompactBlockRepositoryTest { val testedBlocksRange = ListOfCompactBlocksFixture.DEFAULT_FILE_BLOCK_RANGE val blocks = ListOfCompactBlocksFixture.newFlow(testedBlocksRange) - val persistedCount = blockRepository.write(blocks) + val persistedBlocks = blockRepository.write(blocks) parentDirectory.also { assertTrue(it.existsSuspend()) @@ -174,10 +174,10 @@ class FileCompactBlockRepositoryTest { blocksDirectory.also { assertTrue(it.existsSuspend()) - assertEquals(blocks.count(), persistedCount) + assertEquals(blocks.count(), persistedBlocks.size) } - blockRepository.deleteCompactBlockFiles() + blockRepository.deleteAllCompactBlockFiles() parentDirectory.also { assertTrue(it.existsSuspend()) diff --git a/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt new file mode 100644 index 00000000..2788add8 --- /dev/null +++ b/sdk-lib/src/androidTest/java/cash/z/ecc/android/sdk/model/PercentDecimalTest.kt @@ -0,0 +1,38 @@ +package cash.z.ecc.android.sdk.model + +import androidx.test.filters.SmallTest +import org.junit.Test +import kotlin.test.assertTrue + +class PercentDecimalTest { + + @Test(expected = IllegalArgumentException::class) + @SmallTest + fun require_greater_than_zero() { + PercentDecimal(-1.0f) + } + + @Test(expected = IllegalArgumentException::class) + @SmallTest + fun require_less_than_one() { + PercentDecimal(1.5f) + } + + @SmallTest + @Test + fun is_less_than_hundred_percent_test() { + assertTrue(PercentDecimal(0.5f).isLessThanHundredPercent()) + } + + @SmallTest + @Test + fun is_more_than_zero_percent_test() { + assertTrue(PercentDecimal(0.5f).isMoreThanZeroPercent()) + } + + @SmallTest + @Test + fun to_percentage_test() { + assertTrue(PercentDecimal(0.5f).toPercentage() == 50) + } +} diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt index 6caa1035..b06a7a8c 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/SdkSynchronizer.kt @@ -37,6 +37,7 @@ import cash.z.ecc.android.sdk.internal.transaction.TransactionEncoderImpl import cash.z.ecc.android.sdk.jni.RustBackend import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.TransactionOverview import cash.z.ecc.android.sdk.model.TransactionRecipient import cash.z.ecc.android.sdk.model.UnifiedFullViewingKey @@ -193,12 +194,12 @@ class SdkSynchronizer private constructor( override val status = _status.asStateFlow() /** - * Indicates the download progress of the Synchronizer. When progress reaches 100, that - * signals that the Synchronizer is in sync with the network. Balances should be considered + * Indicates the download progress of the Synchronizer. When progress reaches `PercentDecimal.ONE_HUNDRED_PERCENT`, + * that signals that the Synchronizer is in sync with the network. Balances should be considered * inaccurate and outbound transactions should be prevented until this sync is complete. It is * a simplified version of [processorInfo]. */ - override val progress: Flow = processor.progress + override val progress: Flow = processor.progress /** * Indicates the latest information about the blocks that have been processed by the SDK. This @@ -479,18 +480,6 @@ class SdkSynchronizer private constructor( val shouldRefresh = !scannedRange.isNullOrEmpty() || elapsedMillis > (ZcashSdk.POLL_INTERVAL * 5) val reason = if (scannedRange.isNullOrEmpty()) "it's been a while" else "new blocks were scanned" - // TRICKY: - // Keep an eye on this section because there is a potential for concurrent DB - // modification. A change in transactions means a change in balance. Calculating the - // balance requires touching transactions. If both are done in separate threads, the - // database can have issues. On Android, this would manifest as a false positive for a - // "malformed database" exception when the database is not actually corrupt but rather - // locked (i.e. it's a bad error message). - // The balance refresh is done first because it is coroutine-based and will fully - // complete by the time the function returns. - // Ultimately, refreshing the transactions just invalidates views of data that - // already exists and it completes on another thread so it should come after the - // balance refresh is complete. if (shouldRefresh) { Twig.debug { "Triggering utxo refresh since $reason!" } refreshUtxos() diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt index 4f0ad64d..b366edf6 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/Synchronizer.kt @@ -7,6 +7,7 @@ import cash.z.ecc.android.sdk.internal.SaplingParamTool import cash.z.ecc.android.sdk.internal.db.DatabaseCoordinator import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.TransactionOverview import cash.z.ecc.android.sdk.model.TransactionRecipient import cash.z.ecc.android.sdk.model.UnifiedSpendingKey @@ -41,10 +42,11 @@ interface Synchronizer { /** * A flow of progress values, typically corresponding to this Synchronizer downloading blocks. - * Typically, any non- zero value below 100 indicates that progress indicators can be shown and - * a value of 100 signals that progress is complete and any progress indicators can be hidden. + * Typically, any non-zero value below `PercentDecimal.ONE_HUNDRED_PERCENT` indicates that progress indicators can + * be shown and a value of `PercentDecimal.ONE_HUNDRED_PERCENT` signals that progress is complete and any progress + * indicators can be hidden. */ - val progress: Flow + val progress: Flow /** * A flow of processor details, updated every time blocks are processed to include the latest diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt index e75be36b..44ab7463 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/block/CompactBlockProcessor.kt @@ -13,11 +13,7 @@ import cash.z.ecc.android.sdk.exception.RustLayerException import cash.z.ecc.android.sdk.ext.BatchMetrics import cash.z.ecc.android.sdk.ext.ZcashSdk import cash.z.ecc.android.sdk.ext.ZcashSdk.MAX_BACKOFF_INTERVAL -import cash.z.ecc.android.sdk.ext.ZcashSdk.MAX_REORG_SIZE import cash.z.ecc.android.sdk.ext.ZcashSdk.POLL_INTERVAL -import cash.z.ecc.android.sdk.ext.ZcashSdk.RETRIES -import cash.z.ecc.android.sdk.ext.ZcashSdk.REWIND_DISTANCE -import cash.z.ecc.android.sdk.ext.ZcashSdk.SYNC_BATCH_SIZE import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.block.CompactBlockDownloader import cash.z.ecc.android.sdk.internal.ext.retryUpTo @@ -25,7 +21,9 @@ import cash.z.ecc.android.sdk.internal.ext.retryWithBackoff import cash.z.ecc.android.sdk.internal.ext.toHexReversed import cash.z.ecc.android.sdk.internal.isNullOrEmpty import cash.z.ecc.android.sdk.internal.length +import cash.z.ecc.android.sdk.internal.model.BlockBatch import cash.z.ecc.android.sdk.internal.model.DbTransactionOverview +import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ext.from import cash.z.ecc.android.sdk.internal.model.ext.toBlockHeight import cash.z.ecc.android.sdk.internal.repository.DerivedDataRepository @@ -42,6 +40,7 @@ import cash.z.ecc.android.sdk.jni.rewindToHeight import cash.z.ecc.android.sdk.jni.validateCombinedChainOrErrorBlockHeight import cash.z.ecc.android.sdk.model.Account import cash.z.ecc.android.sdk.model.BlockHeight +import cash.z.ecc.android.sdk.model.PercentDecimal import cash.z.ecc.android.sdk.model.UnifiedSpendingKey import cash.z.ecc.android.sdk.model.WalletBalance import cash.z.ecc.android.sdk.model.ZcashNetwork @@ -54,12 +53,16 @@ import co.electriccoin.lightwallet.client.model.Response import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.takeWhile import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.Locale @@ -135,7 +138,7 @@ class CompactBlockProcessor internal constructor( ) private val _state: MutableStateFlow = MutableStateFlow(State.Initialized) - private val _progress = MutableStateFlow(0) + private val _progress = MutableStateFlow(PercentDecimal.ZERO_PERCENT) private val _processorInfo = MutableStateFlow(ProcessorInfo(null, null, null)) private val _networkHeight = MutableStateFlow(null) private val processingMutex = Mutex() @@ -191,7 +194,15 @@ class CompactBlockProcessor internal constructor( @Suppress("LongMethod") suspend fun start() { verifySetup() + updateBirthdayHeight() + + // Clear any undeleted left over block files from previous sync attempts + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = getLastScannedHeight(repository) + ) + Twig.debug { "setup verified. processor starting" } // using do/while makes it easier to execute exactly one loop which helps with testing this processor quickly @@ -237,24 +248,24 @@ class CompactBlockProcessor internal constructor( } is BlockProcessingResult.FailedDeleteBlocks -> { - Twig.warn { + Twig.error { "Failed to delete temporary blocks files from the device disk. It will be retried on the" + " next time, while downloading new blocks." } - // Do nothing. The other phases went correctly. + checkErrorResult(result.failedAtHeight) } - is BlockProcessingResult.FailedDownloadingBlocks -> { + is BlockProcessingResult.FailedDownloadBlocks -> { Twig.error { "Failed while downloading blocks at height: ${result.failedAtHeight}" } checkErrorResult(result.failedAtHeight) } - is BlockProcessingResult.FailedValidationBlocks -> { + is BlockProcessingResult.FailedValidateBlocks -> { Twig.error { "Failed while validating blocks at height: ${result.failedAtHeight}" } checkErrorResult(result.failedAtHeight) } - is BlockProcessingResult.FailedScanningBlocks -> { + is BlockProcessingResult.FailedScanBlocks -> { Twig.error { "Failed while scanning blocks at height: ${result.failedAtHeight}" } checkErrorResult(result.failedAtHeight) } @@ -262,6 +273,10 @@ class CompactBlockProcessor internal constructor( is BlockProcessingResult.Success -> { // Do nothing. We are done. } + + is BlockProcessingResult.DownloadSuccess -> { + // Do nothing. Syncing of blocks is in progress. + } } } } while (_state.value !is State.Stopped) @@ -281,7 +296,7 @@ class CompactBlockProcessor internal constructor( } /** - * Sets the state to [Stopped], which causes the processor loop to exit. + * Sets the state to [State.Stopped], which causes the processor loop to exit. */ suspend fun stop() { runCatching { @@ -352,16 +367,21 @@ class CompactBlockProcessor internal constructor( _progress.value = syncProgress.percentage updateProgress(lastSyncedHeight = syncProgress.lastSyncedHeight) - if (syncProgress.result != BlockProcessingResult.Success || - syncProgress.result != BlockProcessingResult.FailedDeleteBlocks - ) { + // Cancel collecting in case of any unwanted state comes + if (syncProgress.result != BlockProcessingResult.Success) { syncResult = syncProgress.result - return@collect } } if (syncResult != BlockProcessingResult.Success) { - // We should also set the synchronizer status to error here + // Remove persisted but not validated and scanned blocks in case of any failure + val lastScannedHeight = getLastScannedHeight(repository) + downloader.rewindToHeight(lastScannedHeight) + deleteAllBlockFiles( + downloader = downloader, + lastKnownHeight = lastScannedHeight + ) + return syncResult } @@ -380,11 +400,12 @@ class CompactBlockProcessor internal constructor( sealed class BlockProcessingResult { object NoBlocksToProcess : BlockProcessingResult() object Success : BlockProcessingResult() + data class DownloadSuccess(val downloadedBlocks: List?) : BlockProcessingResult() object Reconnecting : BlockProcessingResult() - data class FailedDownloadingBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - data class FailedScanningBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - data class FailedValidationBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() - object FailedDeleteBlocks : BlockProcessingResult() + data class FailedDownloadBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() + data class FailedScanBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() + data class FailedValidateBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() + data class FailedDeleteBlocks(val failedAtHeight: BlockHeight) : BlockProcessingResult() object FailedEnhance : BlockProcessingResult() } @@ -431,24 +452,11 @@ class CompactBlockProcessor internal constructor( lastDownloadedHeight } - ProcessorInfo( + updateProgress( networkBlockHeight = networkBlockHeight, lastSyncedHeight = lastSyncedHeight, - lastSyncRange = null - ).let { initialInfo -> - updateProgress( - networkBlockHeight = initialInfo.networkBlockHeight, - lastSyncedHeight = initialInfo.lastSyncedHeight, - lastSyncRange = if ( - initialInfo.lastSyncedHeight != null && - initialInfo.networkBlockHeight != null - ) { - initialInfo.lastSyncedHeight + 1..initialInfo.networkBlockHeight - } else { - null - } - ) - } + lastSyncRange = lastSyncedHeight + 1..networkBlockHeight + ) return true } @@ -704,14 +712,43 @@ class CompactBlockProcessor internal constructor( companion object { /** - * Request all blocks in the given range and persist them locally for processing, later. + * Default attempts at retrying. + */ + internal const val RETRIES = 5 + + /** + * The theoretical maximum number of blocks in a reorg, due to other bottlenecks in the protocol design. + */ + internal const val MAX_REORG_SIZE = 100 + + /** + * Default size of batches of blocks to request from the compact block service. Then it's also used as a default + * size of batches of blocks to scan via librustzcash. The smaller this number the more granular information can + * be provided about scan state. Unfortunately, it may also lead to a lot of overhead during scanning. + */ + internal const val SYNC_BATCH_SIZE = 10 + + /** + * Default number of blocks to rewind when a chain reorg is detected. This should be large enough to recover + * from the reorg but smaller than the theoretical max reorg size of 100. + */ + internal const val REWIND_DISTANCE = 10 + + /** + * Requests, processes and persists all blocks from the given range. * - * @param syncRange the range of blocks to download. + * @param backend the Rust backend component + * @param downloader the compact block downloader component + * @param repository the derived data repository component + * @param network the network in which the sync mechanism operates + * @param syncRange the range of blocks to download * @param withDownload the flag indicating whether the blocks should also be downloaded and processed, or - * processed existing blocks. + * processed existing blocks + + * @return Flow of BatchSyncProgress sync results */ @VisibleForTesting - @Suppress("MagicNumber", "LongParameterList", "LongMethod") + @Suppress("LongParameterList", "LongMethod") internal suspend fun syncNewBlocks( backend: Backend, downloader: CompactBlockDownloader, @@ -722,104 +759,104 @@ class CompactBlockProcessor internal constructor( ): Flow = flow { if (syncRange.isEmpty()) { Twig.debug { "No blocks to sync" } + emit( + BatchSyncProgress( + percentage = PercentDecimal.ONE_HUNDRED_PERCENT, + lastSyncedHeight = getLastScannedHeight(repository), + result = BlockProcessingResult.Success + ) + ) } else { Twig.debug { "Syncing blocks in range $syncRange" } val batches = getBatchedBlockList(syncRange, network) - // While we run the sync sub-phases for each batch serially now, we'd like to run them in - // parallel to speed up the overall sync process time - batches.forEach { batch -> - Twig.debug { "Sync batch: ${batch.index} of ${batches.size} - $batch" } - Twig.verbose { "Starting to sync batch: $batch" } + batches.asFlow().map { + Twig.debug { "Syncing process starts for batch: $it" } - // Download - if (withDownload) { - downloadBatchOfBlocks( - downloader = downloader, - batch = batch - ).takeIf { it != BlockProcessingResult.Success }?.let { result -> - emit( - BatchSyncProgress( - percentage = (batch.index / batches.size.toFloat() * 100).roundToInt(), - lastSyncedHeight = getLastScannedHeight(repository), - result = result - ) + // Run downloading stage + SyncStageResult( + batch = it, + stageResult = if (withDownload) { + downloadBatchOfBlocks( + downloader = downloader, + batch = it ) - return@flow + } else { + BlockProcessingResult.DownloadSuccess(null) } - } + ) + }.buffer(1).map { downloadStageResult -> + Twig.debug { "Download stage done with result: $downloadStageResult" } - // Validate - validateBatchOfBlocks( - backend = backend, - batch = batch - ).takeIf { it != BlockProcessingResult.Success }?.let { result -> - emit( - BatchSyncProgress( - percentage = (batch.index / batches.size.toFloat() * 100).roundToInt(), - lastSyncedHeight = getLastScannedHeight(repository), - result = result + if (downloadStageResult.stageResult !is BlockProcessingResult.DownloadSuccess) { + // In case of any failure, we just propagate the result + downloadStageResult + } else { + // Enrich batch model with fetched blocks. It's useful for later blocks deletion + downloadStageResult.batch.blocks = downloadStageResult.stageResult.downloadedBlocks + + // Run validation stage + SyncStageResult( + downloadStageResult.batch, + validateBatchOfBlocks( + backend = backend, + batch = downloadStageResult.batch ) ) - return@flow } + }.map { validateResult -> + Twig.debug { "Validation stage done with result: $validateResult" } - // Scan - scanBatchOfBlocks( - backend = backend, - batch = batch - ).takeIf { it != BlockProcessingResult.Success }?.let { result -> - emit( - BatchSyncProgress( - percentage = (batch.index / batches.size.toFloat() * 100).roundToInt(), - lastSyncedHeight = getLastScannedHeight(repository), - result = result + if (validateResult.stageResult != BlockProcessingResult.Success) { + validateResult + } else { + // Run scanning stage + SyncStageResult( + validateResult.batch, + scanBatchOfBlocks( + backend = backend, + batch = validateResult.batch ) ) - return@flow } + }.map { scanResult -> + Twig.debug { "Scan stage done with result: $scanResult" } - // Delete - deleteAllBlockFiles( - downloader = downloader - ).takeIf { it != BlockProcessingResult.Success }?.let { result -> - Twig.warn { "Delete batch block files failed with: $result" } - emit( - BatchSyncProgress( - percentage = (batch.index / batches.size.toFloat() * 100).roundToInt(), - lastSyncedHeight = getLastScannedHeight(repository), - result = result + if (scanResult.stageResult != BlockProcessingResult.Success) { + scanResult + } else { + // Run deletion stage + SyncStageResult( + scanResult.batch, + deleteFilesOfBatchOfBlocks( + downloader = downloader, + batch = scanResult.batch ) ) - // We intentionally do not exit the processing of blocks as the failed deletion phase is not - // critical } + }.onEach { deleteResult -> + Twig.debug { "Deletion stage done with result: $deleteResult" } - Twig.verbose { "Done with batch: $batch" } emit( BatchSyncProgress( - percentage = (batch.index / batches.size.toFloat() * 100).roundToInt(), + percentage = PercentDecimal(deleteResult.batch.index / batches.size.toFloat()), lastSyncedHeight = getLastScannedHeight(repository), - result = BlockProcessingResult.Success + result = deleteResult.stageResult ) ) - } - } - emit( - BatchSyncProgress( - percentage = 100, - lastSyncedHeight = getLastScannedHeight(repository), - result = BlockProcessingResult.Success - ) - ) + Twig.debug { "All sync stages done for the batch: ${deleteResult.batch}" } + }.takeWhile { continuousResult -> + continuousResult.stageResult == BlockProcessingResult.Success + }.collect() + } } private fun getBatchedBlockList( syncRange: ClosedRange, network: ZcashNetwork - ): List { + ): List { val missingBlockCount = syncRange.endInclusive.value - syncRange.start.value + 1 val batchCount = ( missingBlockCount / SYNC_BATCH_SIZE + @@ -831,7 +868,7 @@ class CompactBlockProcessor internal constructor( } var start = syncRange.start - return mutableListOf().apply { + return buildList { for (index in 1..batchCount) { val end = BlockHeight.new( network, @@ -841,7 +878,7 @@ class CompactBlockProcessor internal constructor( ) ) // subtract 1 on the first value because the range is inclusive - add(Batch(index, start..end)) + add(BlockBatch(index, start..end)) start = end + 1 } } @@ -857,9 +894,9 @@ class CompactBlockProcessor internal constructor( @Suppress("MagicNumber") internal suspend fun downloadBatchOfBlocks( downloader: CompactBlockDownloader, - batch: Batch + batch: BlockBatch ): BlockProcessingResult { - var downloadedCount = 0 + var downloadedBlocks = listOf() retryUpTo(RETRIES, { CompactBlockProcessorException.FailedDownload(it) }) { failedAttempts -> if (failedAttempts == 0) { Twig.verbose { "Starting to download batch $batch" } @@ -867,19 +904,19 @@ class CompactBlockProcessor internal constructor( Twig.verbose { "Retrying to download batch $batch after $failedAttempts failure(s)..." } } - downloadedCount = downloader.downloadBlockRange(batch.range) + downloadedBlocks = downloader.downloadBlockRange(batch.range) } - Twig.verbose { "Successfully downloaded batch: $batch of $downloadedCount blocks" } + Twig.verbose { "Successfully downloaded batch: $batch of $downloadedBlocks blocks" } - return if (downloadedCount > 0) { - BlockProcessingResult.Success + return if (downloadedBlocks.isNotEmpty()) { + BlockProcessingResult.DownloadSuccess(downloadedBlocks) } else { - BlockProcessingResult.FailedDownloadingBlocks(batch.range.start) + BlockProcessingResult.FailedDownloadBlocks(batch.range.start) } } @VisibleForTesting - internal suspend fun validateBatchOfBlocks(batch: Batch, backend: Backend): BlockProcessingResult { + internal suspend fun validateBatchOfBlocks(batch: BlockBatch, backend: Backend): BlockProcessingResult { Twig.verbose { "Starting to validate batch $batch" } val result = backend.validateCombinedChainOrErrorBlockHeight(batch.range.length()) @@ -888,33 +925,53 @@ class CompactBlockProcessor internal constructor( Twig.verbose { "Successfully validated batch $batch" } BlockProcessingResult.Success } else { - BlockProcessingResult.FailedValidationBlocks(result) + BlockProcessingResult.FailedValidateBlocks(result) } } @VisibleForTesting - @Suppress("MagicNumber") - internal suspend fun scanBatchOfBlocks(batch: Batch, backend: Backend): BlockProcessingResult { + internal suspend fun scanBatchOfBlocks(batch: BlockBatch, backend: Backend): BlockProcessingResult { val scanResult = backend.scanBlocks(batch.range.length()) return if (scanResult) { Twig.verbose { "Successfully scanned batch $batch" } BlockProcessingResult.Success } else { - BlockProcessingResult.FailedScanningBlocks(batch.range.start) + BlockProcessingResult.FailedScanBlocks(batch.range.start) } } @VisibleForTesting - internal suspend fun deleteAllBlockFiles(downloader: CompactBlockDownloader): BlockProcessingResult { - Twig.verbose { "Starting delete all temporary block files now" } - return if (downloader.compactBlockRepository.deleteCompactBlockFiles()) { + internal suspend fun deleteAllBlockFiles( + downloader: CompactBlockDownloader, + lastKnownHeight: BlockHeight + ): BlockProcessingResult { + Twig.verbose { "Starting to delete all temporary block files" } + return if (downloader.compactBlockRepository.deleteAllCompactBlockFiles()) { Twig.verbose { "Successfully deleted all temporary block files" } BlockProcessingResult.Success } else { - BlockProcessingResult.FailedDeleteBlocks + BlockProcessingResult.FailedDeleteBlocks(lastKnownHeight) } } + @VisibleForTesting + internal suspend fun deleteFilesOfBatchOfBlocks( + batch: BlockBatch, + downloader: CompactBlockDownloader + ): BlockProcessingResult { + Twig.verbose { "Starting to delete temporary block files from batch: $batch" } + + return batch.blocks?.let { blocks -> + val deleted = downloader.compactBlockRepository.deleteCompactBlockFiles(blocks) + if (deleted) { + Twig.verbose { "Successfully deleted all temporary batched block files" } + BlockProcessingResult.Success + } else { + BlockProcessingResult.FailedDeleteBlocks(batch.range.start) + } + } ?: BlockProcessingResult.Success + } + /** * Get the height of the last block that was scanned by this processor. * @@ -979,7 +1036,7 @@ class CompactBlockProcessor internal constructor( * wanted to sync. In most cases, it will be an invalid range because we'd like to sync blocks * that we don't yet have. */ - private suspend fun updateProgress( + private fun updateProgress( networkBlockHeight: BlockHeight? = _processorInfo.value.networkBlockHeight, lastSyncedHeight: BlockHeight? = _processorInfo.value.lastSyncedHeight, lastSyncRange: ClosedRange? = _processorInfo.value.lastSyncRange, @@ -1086,7 +1143,7 @@ class CompactBlockProcessor internal constructor( lastSyncRange = (targetHeight + 1)..currentNetworkBlockHeight ) } - _progress.value = 0 + _progress.value = PercentDecimal.ZERO_PERCENT } else { if (null == currentNetworkBlockHeight) { updateProgress( @@ -1100,7 +1157,7 @@ class CompactBlockProcessor internal constructor( ) } - _progress.value = 0 + _progress.value = PercentDecimal.ZERO_PERCENT if (null != lastSyncedHeight) { val range = (targetHeight + 1)..lastSyncedHeight @@ -1259,7 +1316,7 @@ class CompactBlockProcessor internal constructor( interface ISyncing /** - * [State] for common syncing phase. It starts with downloading new blocks, then validating these blocks + * [State] for common syncing stage. It starts with downloading new blocks, then validating these blocks * and scanning them at the end. * * **Downloading** is when the wallet is actively downloading compact blocks because the latest @@ -1275,7 +1332,7 @@ class CompactBlockProcessor internal constructor( object Syncing : IConnected, ISyncing, State() /** - * [State] for when we are done with syncing the blocks, for now, i.e. all necessary phases done (download, + * [State] for when we are done with syncing the blocks, for now, i.e. all necessary stages done (download, * validate, and scan). */ class Synced(val syncedRange: ClosedRange?) : IConnected, ISyncing, State() @@ -1306,17 +1363,23 @@ class CompactBlockProcessor internal constructor( object Initialized : State() } - internal data class Batch( - val index: Long, - val range: ClosedRange - ) - + /** + * Progress model class for sharing the whole batch sync progress out of the sync process. + */ internal data class BatchSyncProgress( - val percentage: Int, + val percentage: PercentDecimal, val lastSyncedHeight: BlockHeight?, val result: BlockProcessingResult ) + /** + * Progress model class for sharing particular sync stage result internally in the sync process. + */ + private data class SyncStageResult( + val batch: BlockBatch, + val stageResult: BlockProcessingResult + ) + /** * Data class for holding detailed information about the processor. * diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/ext/ZcashSdk.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/ext/ZcashSdk.kt index 2e802ae0..7600d2dd 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/ext/ZcashSdk.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/ext/ZcashSdk.kt @@ -16,11 +16,6 @@ object ZcashSdk { */ val MINERS_FEE = Zatoshi(1_000L) - /** - * The theoretical maximum number of blocks in a reorg, due to other bottlenecks in the protocol design. - */ - const val MAX_REORG_SIZE = 100 - /** * The maximum length of a memo. */ @@ -32,13 +27,6 @@ object ZcashSdk { */ const val EXPIRY_OFFSET = 20 - /** - * Default size of batches of blocks to request from the compact block service. Then it's also used as a default - * size of batches of blocks to scan via librustzcash. The smaller this number the more granular information can be - * provided about scan state. Unfortunately, it may also lead to a lot of overhead during scanning. - */ - const val SYNC_BATCH_SIZE = 10 - /** * Default amount of time, in milliseconds, to poll for new blocks. Typically, this should be about half the average * block time. @@ -50,23 +38,12 @@ object ZcashSdk { */ const val BLOCK_INTERVAL_MILLIS = 75_000L - /** - * Default attempts at retrying. - */ - const val RETRIES = 5 - /** * The default maximum amount of time to wait during retry backoff intervals. Failed loops will never wait longer * than this before retyring. */ const val MAX_BACKOFF_INTERVAL = 600_000L - /** - * Default number of blocks to rewind when a chain reorg is detected. This should be large enough to recover from - * the reorg but smaller than the theoretical max reorg size of 100. - */ - const val REWIND_DISTANCE = 10 - /** * The default memo to use when shielding transparent funds. */ diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt index d2e2a379..c282ade4 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/block/CompactBlockDownloader.kt @@ -3,6 +3,7 @@ package cash.z.ecc.android.sdk.internal.block import cash.z.ecc.android.sdk.exception.LightWalletException import cash.z.ecc.android.sdk.internal.Twig import cash.z.ecc.android.sdk.internal.ext.retryUpTo +import cash.z.ecc.android.sdk.internal.model.JniBlockMeta import cash.z.ecc.android.sdk.internal.model.ext.from import cash.z.ecc.android.sdk.internal.repository.CompactBlockRepository import cash.z.ecc.android.sdk.model.BlockHeight @@ -47,10 +48,11 @@ open class CompactBlockDownloader private constructor(val compactBlockRepository * @param heightRange the inclusive range of heights to request. For example 10..20 would * request 11 blocks (including block 10 and block 20). * @throws LightWalletException.DownloadBlockException if any error while downloading the blocks occurs - * @return Number of blocks, which were successfully written to storage + * @return List of JniBlockMeta objects, which describe the original CompactBlock objects, which were just + * downloaded and persisted on the device disk */ @Throws(LightWalletException.DownloadBlockException::class) - suspend fun downloadBlockRange(heightRange: ClosedRange): Int { + suspend fun downloadBlockRange(heightRange: ClosedRange): List { val filteredFlow = lightWalletClient.getBlockRange( BlockHeightUnsafe.from(heightRange.start)..BlockHeightUnsafe.from(heightRange.endInclusive) ).onEach { response -> diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt new file mode 100644 index 00000000..aaf27fff --- /dev/null +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/model/BlockBatch.kt @@ -0,0 +1,9 @@ +package cash.z.ecc.android.sdk.internal.model + +import cash.z.ecc.android.sdk.model.BlockHeight + +internal data class BlockBatch( + val index: Long, + val range: ClosedRange, + var blocks: List? = null +) diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/CompactBlockRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/CompactBlockRepository.kt index 42d065bb..7bbee6c0 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/CompactBlockRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/repository/CompactBlockRepository.kt @@ -12,32 +12,42 @@ interface CompactBlockRepository { /** * Gets the highest block that is currently stored. * - * @return the latest block height. + * @return the latest block height */ suspend fun getLatestHeight(): BlockHeight? /** * Fetch the compact block for the given height, if it exists. * - * @return the compact block or null when it did not exist. + * @param height of the block we are looking for + * @return the compact block summary object or null when it did not exist */ suspend fun findCompactBlock(height: BlockHeight): JniBlockMeta? /** * This function is supposed to be used once the whole blocks sync process done. It removes all the temporary - * block files from the device disk together with theirs parent directory. + * block files from the device disk. * - * @return true when all block files are deleted, false only if the deletion fails + * @return true when all block files are deleted or do not exist, false only if the deletion fails */ - suspend fun deleteCompactBlockFiles(): Boolean + suspend fun deleteAllCompactBlockFiles(): Boolean + + /** + * This function is supposed to be used continuously while sync process is in progress. It removes all the temporary + * block files from the given list of blocks. + * + * @param blocks list of compact block summary objects of blocks to delete + * @return true when all block files from the list are deleted or do not exist, false only if the deletion fails + */ + suspend fun deleteCompactBlockFiles(blocks: List): Boolean /** * Write the given flow of blocks to this store, which may be anything from an in-memory cache to a DB. * - * @param blocks Flow of compact blocks to persist. - * @return Flow of number of blocks that were written. + * @param blocks Flow of compact blocks to persist + * @return list of compact block summary objects of blocks that were written */ - suspend fun write(blocks: Flow): Int + suspend fun write(blocks: Flow): List /** * Remove every block above the given height. diff --git a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt index 17400e0a..c3a62cdb 100644 --- a/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/internal/storage/block/FileCompactBlockRepository.kt @@ -33,8 +33,8 @@ internal class FileCompactBlockRepository( override suspend fun findCompactBlock(height: BlockHeight) = rustBackend.findBlockMetadata(height) - override suspend fun write(blocks: Flow): Int { - var totalBlocksWritten = 0 + override suspend fun write(blocks: Flow): List { + val processingBlocks = mutableListOf() val metaDataBuffer = mutableListOf() blocks.collect { block -> val tmpFile = block.createTemporaryFile(blocksDirectory) @@ -49,32 +49,30 @@ internal class FileCompactBlockRepository( } if (metaDataBuffer.isBufferFull()) { - val blocksWritten = writeAndClearBuffer(metaDataBuffer) - totalBlocksWritten += blocksWritten + processingBlocks.addAll(metaDataBuffer) + writeAndClearBuffer(metaDataBuffer) } } if (metaDataBuffer.isNotEmpty()) { - val blocksWritten = writeAndClearBuffer(metaDataBuffer) - totalBlocksWritten += blocksWritten + processingBlocks.addAll(metaDataBuffer) + writeAndClearBuffer(metaDataBuffer) } - return totalBlocksWritten + return processingBlocks } /* * Write block metadata to storage when the buffer is full or when we reached the current range end. */ - private suspend fun writeAndClearBuffer(metaDataBuffer: MutableList): Int { + private suspend fun writeAndClearBuffer(metaDataBuffer: MutableList) { rustBackend.writeBlockMetadata(metaDataBuffer) - val blocksWrittenCount = metaDataBuffer.size metaDataBuffer.clear() - return blocksWrittenCount } override suspend fun rewindTo(height: BlockHeight) = rustBackend.rewindBlockMetadataToHeight(height) - override suspend fun deleteCompactBlockFiles(): Boolean { + override suspend fun deleteAllCompactBlockFiles(): Boolean { Twig.verbose { "Deleting all blocks from directory ${blocksDirectory.path}" } if (blocksDirectory.existsSuspend()) { @@ -92,6 +90,24 @@ internal class FileCompactBlockRepository( return true } + override suspend fun deleteCompactBlockFiles(blocks: List): Boolean { + Twig.verbose { "Deleting ${blocks.size} blocks from directory ${blocksDirectory.path}" } + + if (blocksDirectory.existsSuspend()) { + blocks.forEach { block -> + val blockFile = block.getFile(blocksDirectory) + if (!blockFile.existsSuspend()) { + return@forEach // aka continue + } + val deleted = blockFile.deleteSuspend() + if (!deleted) { + return false + } + } + } + return true + } + companion object { /** * The name of the directory for downloading blocks @@ -145,6 +161,16 @@ private fun CompactBlockUnsafe.toJniMetaData(): JniBlockMeta { return JniBlockMeta.new(this) } +private fun JniBlockMeta.createFilename(): String { + val hashHex = hash.toHexReversed() + return "$height-$hashHex${FileCompactBlockRepository.BLOCK_FILENAME_SUFFIX}" +} + +@VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) +private fun JniBlockMeta.getFile(blocksDirectory: File): File { + return File(blocksDirectory, createFilename()) +} + private fun CompactBlockUnsafe.createFilename(): String { val hashHex = hash.toHexReversed() return "$height-$hashHex${FileCompactBlockRepository.BLOCK_FILENAME_SUFFIX}" diff --git a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt similarity index 67% rename from sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt rename to sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt index 2096b08d..27c109ab 100644 --- a/sdk-incubator-lib/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt +++ b/sdk-lib/src/main/java/cash/z/ecc/android/sdk/model/PercentDecimal.kt @@ -10,9 +10,16 @@ value class PercentDecimal(val decimal: Float) { require(decimal <= MAX) } + fun isLessThanHundredPercent(): Boolean = decimal < MAX + + fun isMoreThanZeroPercent(): Boolean = decimal > MIN + + @Suppress("MagicNumber") + fun toPercentage(): Int = (decimal * 100).toInt() + companion object { - const val MIN = 0.0f - const val MAX = 1.0f + private const val MIN = 0.0f + private const val MAX = 1.0f val ZERO_PERCENT = PercentDecimal(MIN) val ONE_HUNDRED_PERCENT = PercentDecimal(MAX)