From 56e4dda40f8aed9ea8c56b47d717f0dbfc796c5b Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:04:18 -0400 Subject: [PATCH 01/18] Update dependencies. --- build.gradle | 8 ++++---- gradle.properties | 1 + gradle/wrapper/gradle-wrapper.properties | 4 ++-- samples/memo/build.gradle | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index d6279632..b5d5901b 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ buildscript { ], 'grpc':'1.19.0', 'kotlin': '1.3.21', - 'coroutines': '1.1.1', + 'coroutines': '1.3.0-M1', 'junitJupiter': '5.5.0-M1' ] repositories { @@ -22,7 +22,7 @@ buildscript { } } dependencies { - classpath 'com.android.tools.build:gradle:3.5.0-alpha09' + classpath 'com.android.tools.build:gradle:3.5.0-beta03' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${versions.kotlin}" classpath "org.jetbrains.kotlin:kotlin-allopen:${versions.kotlin}" classpath "org.jetbrains.dokka:dokka-gradle-plugin:0.9.18" @@ -75,12 +75,12 @@ android { debug { // for test builds, which exceed the dex limit because they pull in things like mockito and grpc-testing multiDexEnabled true - minifyEnabled true + minifyEnabled false proguardFiles getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro" } release { multiDexEnabled false - minifyEnabled true + minifyEnabled false proguardFiles getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro" } } diff --git a/gradle.properties b/gradle.properties index 23339e0d..dbd89ae4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,3 +19,4 @@ android.useAndroidX=true android.enableJetifier=true # Kotlin code style for this project: "official" or "obsolete": kotlin.code.style=official +android.enableR8=false diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3e0fd9f9..da3cadf1 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Tue Mar 12 10:04:58 EDT 2019 +#Fri Jun 07 02:04:27 EDT 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip diff --git a/samples/memo/build.gradle b/samples/memo/build.gradle index 2820cae4..b39dc023 100644 --- a/samples/memo/build.gradle +++ b/samples/memo/build.gradle @@ -11,7 +11,7 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:3.5.0-alpha12' + classpath 'com.android.tools.build:gradle:3.5.0-beta03' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${versions.kotlin}" // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files From b14401eaeb39f167d0ebccf456bb93ab5ce79997 Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:15:44 -0400 Subject: [PATCH 02/18] Twig: Add timing features and prevent duplicate tags. --- src/main/java/cash/z/wallet/sdk/data/Twig.kt | 31 +++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/java/cash/z/wallet/sdk/data/Twig.kt b/src/main/java/cash/z/wallet/sdk/data/Twig.kt index 5cb31417..f7661f9a 100644 --- a/src/main/java/cash/z/wallet/sdk/data/Twig.kt +++ b/src/main/java/cash/z/wallet/sdk/data/Twig.kt @@ -1,6 +1,7 @@ package cash.z.wallet.sdk.data import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CopyOnWriteArraySet internal typealias Leaf = String @@ -31,6 +32,11 @@ interface Twig { * Clip a leaf from the bush. Clipped leaves no longer appear in logs. */ fun clip(leaf: Leaf) = Bush.leaves.remove(leaf) + + /** + * Clip all leaves from the bush. + */ + fun prune() = Bush.leaves.clear() } } @@ -45,7 +51,7 @@ interface Twig { */ object Bush { var trunk: Twig = SilentTwig() - val leaves: MutableList = CopyOnWriteArrayList() + val leaves: MutableSet = CopyOnWriteArraySet() } /** @@ -54,7 +60,12 @@ object Bush { inline fun twig(message: String) = Bush.trunk.twig(message) /** - * Times a tiny log task. Execute the block of code with some twigging around the outside. + * Times a tiny log. + */ +inline fun twig(logMessage: String, block: () -> R): R = Bush.trunk.twig(logMessage, block) + +/** + * Meticulously times a tiny task. */ inline fun twigTask(logMessage: String, block: () -> R): R = Bush.trunk.twigTask(logMessage, block) @@ -98,6 +109,17 @@ open class CompositeTwig(private val twigBundle: MutableList) : Twig { } } +/** + * Times a tiny log. Execute the block of code on the clock. + */ +inline fun Twig.twig(logMessage: String, block: () -> R): R { + val start = System.currentTimeMillis() + val result = block() + val elapsed = (System.currentTimeMillis() - start) + twig("$logMessage | ${elapsed}ms") + return result +} + /** * A tiny log task. Execute the block of code with some twigging around the outside. For silent twigs, this adds a small * amount of overhead at the call site but still avoids logging. @@ -107,10 +129,11 @@ open class CompositeTwig(private val twigBundle: MutableList) : Twig { * (otherwise the function and its "block" param would have to suspend) */ inline fun Twig.twigTask(logMessage: String, block: () -> R): R { - val start = System.nanoTime() twig("$logMessage - started | on thread ${Thread.currentThread().name})") + val start = System.nanoTime() val result = block() - twig("$logMessage - completed | in ${System.nanoTime() - start}ms" + + val elapsed = ((System.nanoTime() - start)/1e6) + twig("$logMessage - completed | in $elapsed ms" + " on thread ${Thread.currentThread().name}") return result } From 4c635362a8a208c2422994084f6bdf3c9db2393e Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:24:52 -0400 Subject: [PATCH 03/18] Add logic for handling reorgs and simplify synchronization. --- .../z/wallet/sdk/block/CompactBlockDbStore.kt | 46 ++++ .../sdk/block/CompactBlockDownloader.kt | 41 ++++ .../wallet/sdk/block/CompactBlockProcessor.kt | 209 +++++++++++++++++ .../z/wallet/sdk/block/CompactBlockStore.kt | 26 +++ .../cash/z/wallet/sdk/dao/CompactBlockDao.kt | 8 +- .../wallet/sdk/data/CompactBlockProcessor.kt | 122 ---------- .../z/wallet/sdk/data/CompactBlockStream.kt | 214 ------------------ .../z/wallet/sdk/data/MockSynchronizer.kt | 17 +- .../cash/z/wallet/sdk/data/SdkSynchronizer.kt | 135 ++--------- .../cash/z/wallet/sdk/data/Synchronizer.kt | 9 - .../sdk/service/LightWalletGrpcService.kt | 56 +++++ .../wallet/sdk/service/LightWalletService.kt | 28 +++ .../sdk/block/CompactBlockProcessorTest.kt | 153 +++++++++++++ .../sdk/data/CompactBlockDownloaderTest.kt | 173 -------------- 14 files changed, 589 insertions(+), 648 deletions(-) create mode 100644 src/main/java/cash/z/wallet/sdk/block/CompactBlockDbStore.kt create mode 100644 src/main/java/cash/z/wallet/sdk/block/CompactBlockDownloader.kt create mode 100644 src/main/java/cash/z/wallet/sdk/block/CompactBlockProcessor.kt create mode 100644 src/main/java/cash/z/wallet/sdk/block/CompactBlockStore.kt delete mode 100644 src/main/java/cash/z/wallet/sdk/data/CompactBlockProcessor.kt delete mode 100644 src/main/java/cash/z/wallet/sdk/data/CompactBlockStream.kt create mode 100644 src/main/java/cash/z/wallet/sdk/service/LightWalletGrpcService.kt create mode 100644 src/main/java/cash/z/wallet/sdk/service/LightWalletService.kt create mode 100644 src/test/java/cash/z/wallet/sdk/block/CompactBlockProcessorTest.kt delete mode 100644 src/test/java/cash/z/wallet/sdk/data/CompactBlockDownloaderTest.kt diff --git a/src/main/java/cash/z/wallet/sdk/block/CompactBlockDbStore.kt b/src/main/java/cash/z/wallet/sdk/block/CompactBlockDbStore.kt new file mode 100644 index 00000000..9f4fee83 --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/block/CompactBlockDbStore.kt @@ -0,0 +1,46 @@ +package cash.z.wallet.sdk.block + +import android.content.Context +import androidx.room.Room +import androidx.room.RoomDatabase +import cash.z.wallet.sdk.dao.CompactBlockDao +import cash.z.wallet.sdk.db.CompactBlockDb +import cash.z.wallet.sdk.entity.CompactBlock +import cash.z.wallet.sdk.ext.SAPLING_ACTIVATION_HEIGHT +import kotlinx.coroutines.Dispatchers.IO +import kotlinx.coroutines.withContext + +class CompactBlockDbStore( + applicationContext: Context, + cacheDbName: String +) : CompactBlockStore { + + private val cacheDao: CompactBlockDao + private val cacheDb: CompactBlockDb + + init { + cacheDb = createCompactBlockCacheDb(applicationContext, cacheDbName) + cacheDao = cacheDb.complactBlockDao() + } + + private fun createCompactBlockCacheDb(applicationContext: Context, cacheDbName: String): CompactBlockDb { + return Room.databaseBuilder(applicationContext, CompactBlockDb::class.java, cacheDbName) + .setJournalMode(RoomDatabase.JournalMode.TRUNCATE) + // this is a simple cache of blocks. destroying the db should be benign + .fallbackToDestructiveMigration() + .build() + } + + override suspend fun getLatestHeight(): Int = withContext(IO) { + val lastBlock = Math.max(0, cacheDao.latestBlockHeight() - 1) + if (lastBlock < SAPLING_ACTIVATION_HEIGHT) -1 else lastBlock + } + + override suspend fun write(result: List) = withContext(IO) { + cacheDao.insert(result) + } + + override suspend fun rewindTo(height: Int) = withContext(IO) { + cacheDao.rewindTo(height) + } +} \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/block/CompactBlockDownloader.kt b/src/main/java/cash/z/wallet/sdk/block/CompactBlockDownloader.kt new file mode 100644 index 00000000..d314cb2e --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/block/CompactBlockDownloader.kt @@ -0,0 +1,41 @@ +package cash.z.wallet.sdk.block + +import cash.z.wallet.sdk.data.twig +import cash.z.wallet.sdk.service.LightWalletService +import kotlinx.coroutines.Dispatchers.IO +import kotlinx.coroutines.withContext + +/** + * Serves as a source of compact blocks received from the light wallet server. Once started, it will use the given + * lightwallet service to request all the appropriate blocks and compact block store to persist them. By delegating to + * these dependencies, the downloader remains agnostic to the particular implementation of how to retrieve and store + * data; although, by default the SDK uses gRPC and SQL. + * + * @property lightwalletService the service used for requesting compact blocks + * @property compactBlockStore responsible for persisting the compact blocks that are received + */ +open class CompactBlockDownloader( + val lightwalletService: LightWalletService, + val compactBlockStore: CompactBlockStore +) { + + suspend fun downloadBlockRange(heightRange: IntRange) = withContext(IO) { + val result = lightwalletService.getBlockRange(heightRange) + compactBlockStore.write(result) + } + + suspend fun rewindTo(height: Int) = withContext(IO) { + // TODO: cancel anything in flight + compactBlockStore.rewindTo(height) + } + + suspend fun getLatestBlockHeight() = withContext(IO) { + lightwalletService.getLatestBlockHeight() + } + + suspend fun getLastDownloadedHeight() = withContext(IO) { + compactBlockStore.getLatestHeight() + } + +} + diff --git a/src/main/java/cash/z/wallet/sdk/block/CompactBlockProcessor.kt b/src/main/java/cash/z/wallet/sdk/block/CompactBlockProcessor.kt new file mode 100644 index 00000000..3b42d922 --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/block/CompactBlockProcessor.kt @@ -0,0 +1,209 @@ +package cash.z.wallet.sdk.block + +import androidx.annotation.VisibleForTesting +import cash.z.wallet.sdk.annotation.OpenForTesting +import cash.z.wallet.sdk.data.TransactionRepository +import cash.z.wallet.sdk.data.Twig +import cash.z.wallet.sdk.data.twig +import cash.z.wallet.sdk.exception.CompactBlockProcessorException +import cash.z.wallet.sdk.ext.* +import cash.z.wallet.sdk.jni.RustBackend +import cash.z.wallet.sdk.jni.RustBackendWelding +import kotlinx.coroutines.Dispatchers.IO +import kotlinx.coroutines.channels.ConflatedBroadcastChannel +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withContext +import java.io.File +import java.util.concurrent.atomic.AtomicInteger + +/** + * Responsible for processing the compact blocks that are received from the lightwallet server. This class encapsulates + * all the business logic required to validate and scan the blockchain and is therefore tightly coupled with + * librustzcash. + */ +@OpenForTesting +class CompactBlockProcessor( + internal val config: ProcessorConfig, + internal val downloader: CompactBlockDownloader, + private val repository: TransactionRepository, + private val rustBackend: RustBackendWelding = RustBackend() +) { + private val progressChannel = ConflatedBroadcastChannel() + private var isStopped = false + private val consecutiveErrors = AtomicInteger(0) + + fun progress(): ReceiveChannel = progressChannel.openSubscription() + + /** + * Download compact blocks, verify and scan them. + */ + suspend fun start() = withContext(IO) { + twig("processor starting") + validateConfig() + + // using do/while makes it easier to execute exactly one loop which helps with testing this processor quickly + do { + retryUpTo(config.retries) { + val result = processNewBlocks() + // immediately process again after failures in order to download new blocks right away + if (result < 0) { + consecutiveErrors.set(0) + twig("Successfully processed new blocks. Sleeping for ${config.blockPollFrequencyMillis}ms") + delay(config.blockPollFrequencyMillis) + } else { + if(consecutiveErrors.get() >= config.retries) { + val errorMessage = "ERROR: unable to resolve reorg at height $result after ${consecutiveErrors.get()} correction attempts!" + fail(CompactBlockProcessorException.FailedReorgRepair(errorMessage)) + } else { + handleChainError(result) + } + consecutiveErrors.getAndIncrement() + } + } + } while (isActive && !isStopped) + twig("processor complete") + stop() + } + + /** + * Validate the config to expose a common pitfall. + */ + private fun validateConfig() { + if(!config.cacheDbPath.contains(File.separator)) + throw CompactBlockProcessorException.FileInsteadOfPath(config.cacheDbPath) + if(!config.dataDbPath.contains(File.separator)) + throw CompactBlockProcessorException.FileInsteadOfPath(config.dataDbPath) + } + + fun stop() { + isStopped = true + } + + fun fail(error: Throwable) { + stop() + twig("${error.message}") + throw error + } + + /** + * Process new blocks returning false whenever an error was found. + * + * @return -1 when processing was successful and did not encounter errors during validation or scanning. Otherwise + * return the block height where an error was found. + */ + private suspend fun processNewBlocks(): Int { + twig("beginning to process new blocks...") + + // define ranges + val latestBlockHeight = downloader.getLatestBlockHeight() + val lastDownloadedHeight = Math.max(getLastDownloadedHeight(), SAPLING_ACTIVATION_HEIGHT - 1) + val lastScannedHeight = getLastScannedHeight() + + twig("latestBlockHeight: $latestBlockHeight\tlastDownloadedHeight: $lastDownloadedHeight" + + "\tlastScannedHeight: $lastScannedHeight") + + // as long as the database has the sapling tree (like when it's initialized from a checkpoint) we can avoid + // downloading earlier blocks so take the larger of these two numbers + val rangeToDownload = (Math.max(lastDownloadedHeight, lastScannedHeight) + 1)..latestBlockHeight + val rangeToScan = (lastScannedHeight + 1)..latestBlockHeight + + downloadNewBlocks(rangeToDownload) + val error = validateNewBlocks(rangeToScan) + return if (error < 0) { + scanNewBlocks(rangeToScan) + -1 // TODO: in theory scan should not fail when validate succeeds but maybe consider returning the failed block height whenever scan does fail + } else { + error + } + } + + + @VisibleForTesting //allow mocks to verify how this is called, rather than the downloader, which is more complex + internal suspend fun downloadNewBlocks(range: IntRange) { + if (range.isEmpty()) { + twig("no blocks to download") + return + } + Twig.sprout("downloading") + twig("downloading blocks in range $range") + + var downloadedBlockHeight = range.start + val missingBlockCount = range.last - range.first + 1 + val batches = (missingBlockCount / config.downloadBatchSize + + (if (missingBlockCount.rem(config.downloadBatchSize) == 0) 0 else 1)) + var progress: Int + twig("found $missingBlockCount missing blocks, downloading in $batches batches of ${config.downloadBatchSize}...") + for (i in 1..batches) { + retryUpTo(config.retries) { + val end = Math.min(range.first + (i * config.downloadBatchSize), range.last + 1) + val batchRange = downloadedBlockHeight..(end - 1) + twig("downloaded $batchRange (batch $i of $batches)") { + downloader.downloadBlockRange(batchRange) + } + progress = Math.round(i / batches.toFloat() * 100) + progressChannel.send(progress) + downloadedBlockHeight = end + } + } + Twig.clip("downloading") + } + + private fun validateNewBlocks(range: IntRange?): Int { + if (range?.isEmpty() != false) { + twig("no blocks to validate: $range") + return -1 + } + Twig.sprout("validating") + twig("validating blocks in range $range") + val result = rustBackend.validateCombinedChain(config.cacheDbPath, config.dataDbPath) + Twig.clip("validating") + return result + } + + private fun scanNewBlocks(range: IntRange?): Boolean { + if (range?.isEmpty() != false) { + twig("no blocks to scan") + return true + } + Twig.sprout("scanning") + twig("scanning blocks in range $range") + val result = rustBackend.scanBlocks(config.cacheDbPath, config.dataDbPath) + Twig.clip("scanning") + return result + } + + private suspend fun handleChainError(errorHeight: Int) = withContext(IO) { + val lowerBound = determineLowerBound(errorHeight) + twig("handling chain error at $errorHeight by rewinding to block $lowerBound") + rustBackend.rewindToHeight(config.dataDbPath, lowerBound) + downloader.rewindTo(lowerBound) + } + + private fun determineLowerBound(errorHeight: Int): Int { + val offset = Math.min(MAX_REORG_SIZE, config.rewindDistance * (consecutiveErrors.get() + 1)) + return Math.max(errorHeight - offset, SAPLING_ACTIVATION_HEIGHT) + } + + suspend fun getLastDownloadedHeight() = withContext(IO) { + downloader.getLastDownloadedHeight() + } + + suspend fun getLastScannedHeight() = withContext(IO) { + repository.lastScannedHeight() + } +} + +/** + * @property cacheDbPath absolute file path of the DB where raw, unprocessed compact blocks are stored. + * @property dataDbPath absolute file path of the DB where all information derived from the cache DB is stored. + */ +data class ProcessorConfig( + val cacheDbPath: String = "", + val dataDbPath: String = "", + val downloadBatchSize: Int = DEFAULT_BATCH_SIZE, + val blockPollFrequencyMillis: Long = DEFAULT_POLL_INTERVAL, + val retries: Int = DEFAULT_RETRIES, + val rewindDistance: Int = DEFAULT_REWIND_DISTANCE +) \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/block/CompactBlockStore.kt b/src/main/java/cash/z/wallet/sdk/block/CompactBlockStore.kt new file mode 100644 index 00000000..b870ba66 --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/block/CompactBlockStore.kt @@ -0,0 +1,26 @@ +package cash.z.wallet.sdk.block + +import cash.z.wallet.sdk.entity.CompactBlock + +/** + * Interface for storing compact blocks. + */ +interface CompactBlockStore { + /** + * Gets the highest block that is currently stored. + */ + suspend fun getLatestHeight(): Int + + /** + * Write the given blocks to this store, which may be anything from an in-memory cache to a DB. + */ + suspend fun write(result: List) + + /** + * Remove every block above and including the given height. + * + * After this operation, the data store will look the same as one that has not yet stored the given block height. + * Meaning, if max height is 100 block and rewindTo(50) is called, then the highest block remaining will be 49. + */ + suspend fun rewindTo(height: Int) +} \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/dao/CompactBlockDao.kt b/src/main/java/cash/z/wallet/sdk/dao/CompactBlockDao.kt index 560b9fb6..5530b868 100644 --- a/src/main/java/cash/z/wallet/sdk/dao/CompactBlockDao.kt +++ b/src/main/java/cash/z/wallet/sdk/dao/CompactBlockDao.kt @@ -8,9 +8,15 @@ import cash.z.wallet.sdk.entity.CompactBlock @Dao interface CompactBlockDao { - @Insert(onConflict = OnConflictStrategy.IGNORE) + @Insert(onConflict = OnConflictStrategy.REPLACE) fun insert(block: CompactBlock) + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insert(block: List) + + @Query("DELETE FROM compactblocks WHERE height >= :height") + fun rewindTo(height: Int) + @Query("SELECT MAX(height) FROM compactblocks") fun latestBlockHeight(): Int } \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/data/CompactBlockProcessor.kt b/src/main/java/cash/z/wallet/sdk/data/CompactBlockProcessor.kt deleted file mode 100644 index 14aa3ddc..00000000 --- a/src/main/java/cash/z/wallet/sdk/data/CompactBlockProcessor.kt +++ /dev/null @@ -1,122 +0,0 @@ -package cash.z.wallet.sdk.data - -import android.content.Context -import androidx.room.Room -import androidx.room.RoomDatabase -import cash.z.wallet.sdk.dao.CompactBlockDao -import cash.z.wallet.sdk.db.CompactBlockDb -import cash.z.wallet.sdk.exception.CompactBlockProcessorException -import cash.z.wallet.sdk.jni.RustBackend -import cash.z.wallet.sdk.jni.RustBackendWelding -import cash.z.wallet.sdk.rpc.CompactFormats -import kotlinx.coroutines.Dispatchers.IO -import kotlinx.coroutines.channels.ReceiveChannel -import kotlinx.coroutines.isActive -import kotlinx.coroutines.withContext -import java.io.File - -/** - * Responsible for processing the blocks on the stream. Saves them to the cacheDb and periodically scans for transactions. - * - * @property applicationContext used to connect to the DB on the device. No reference is kept beyond construction. - */ -class CompactBlockProcessor( - applicationContext: Context, - val rustBackend: RustBackendWelding = RustBackend(), - cacheDbName: String = DEFAULT_CACHE_DB_NAME, - dataDbName: String = DEFAULT_DATA_DB_NAME -) { - - internal val cacheDao: CompactBlockDao - private val cacheDb: CompactBlockDb - private val cacheDbPath: String - private val dataDbPath: String - - val dataDbExists get() = File(dataDbPath).exists() - val cachDbExists get() = File(cacheDbPath).exists() - - init { - cacheDb = createCompactBlockCacheDb(applicationContext, cacheDbName) - cacheDao = cacheDb.complactBlockDao() - cacheDbPath = applicationContext.getDatabasePath(cacheDbName).absolutePath - dataDbPath = applicationContext.getDatabasePath(dataDbName).absolutePath - } - - private fun createCompactBlockCacheDb(applicationContext: Context, cacheDbName: String): CompactBlockDb { - return Room.databaseBuilder(applicationContext, CompactBlockDb::class.java, cacheDbName) - .setJournalMode(RoomDatabase.JournalMode.TRUNCATE) - // this is a simple cache of blocks. destroying the db should be benign - .fallbackToDestructiveMigration() - .build() - } - - /** - * Save blocks and periodically scan them. - */ - suspend fun processBlocks(incomingBlocks: ReceiveChannel) = withContext(IO) { - ensureDataDb() - twigTask("processing blocks") { - var lastScanTime = System.currentTimeMillis() - var hasScanned = false - while (isActive && !incomingBlocks.isClosedForReceive) { - twig("awaiting next block") - val nextBlock = incomingBlocks.receive() - val nextBlockHeight = nextBlock.height - twig("received block with height ${nextBlockHeight} on thread ${Thread.currentThread().name}") - cacheDao.insert(cash.z.wallet.sdk.entity.CompactBlock(nextBlockHeight.toInt(), nextBlock.toByteArray())) - if (shouldScanBlocks(lastScanTime, hasScanned)) { - twig("last block prior to scan ${nextBlockHeight}") - scanBlocks() - lastScanTime = System.currentTimeMillis() - hasScanned = true - } - } - cacheDb.close() - } - } - - private fun ensureDataDb() { - if (!dataDbExists) throw CompactBlockProcessorException.DataDbMissing(dataDbPath) - } - - private fun shouldScanBlocks(lastScanTime: Long, hasScanned: Boolean): Boolean { - val deltaTime = System.currentTimeMillis() - lastScanTime - twig("${deltaTime}ms since last scan. Have we ever scanned? $hasScanned") - return (!hasScanned && deltaTime > INITIAL_SCAN_DELAY) - || deltaTime > SCAN_FREQUENCY - } - - suspend fun scanBlocks() = withContext(IO) { - Twig.sprout("scan") - twigTask("scanning blocks") { - if (isActive) { - try { - rustBackend.scanBlocks(cacheDbPath, dataDbPath) - } catch (t: Throwable) { - twig("error while scanning blocks: $t") - } - } - } - Twig.clip("scan") - } - - /** - * Returns the height of the last processed block or -1 if no blocks have been processed. - */ - suspend fun lastProcessedBlock(): Int = withContext(IO) { - val lastBlock = Math.max(0, cacheDao.latestBlockHeight() - 1) - if (lastBlock < SAPLING_ACTIVATION_HEIGHT) -1 else lastBlock - } - - companion object { - const val DEFAULT_CACHE_DB_NAME = "DownloadedCompactBlocks.db" - const val DEFAULT_DATA_DB_NAME = "CompactBlockScanResults.db" - - /** Default amount of time to synchronize before initiating the first scan. This allows time to download a few blocks. */ - const val INITIAL_SCAN_DELAY = 3000L - /** Minimum amount of time between scans. The frequency with which we check whether the block height has changed and, if so, trigger a scan */ - const val SCAN_FREQUENCY = 75_000L - // TODO: find a better home for this constant - const val SAPLING_ACTIVATION_HEIGHT = 280_000 - } -} diff --git a/src/main/java/cash/z/wallet/sdk/data/CompactBlockStream.kt b/src/main/java/cash/z/wallet/sdk/data/CompactBlockStream.kt deleted file mode 100644 index 95c2ce9c..00000000 --- a/src/main/java/cash/z/wallet/sdk/data/CompactBlockStream.kt +++ /dev/null @@ -1,214 +0,0 @@ -package cash.z.wallet.sdk.data - -import cash.z.wallet.sdk.exception.CompactBlockStreamException -import cash.z.wallet.sdk.ext.toBlockRange -import cash.z.wallet.sdk.rpc.CompactFormats.CompactBlock -import cash.z.wallet.sdk.rpc.CompactTxStreamerGrpc -import cash.z.wallet.sdk.rpc.CompactTxStreamerGrpc.CompactTxStreamerBlockingStub -import cash.z.wallet.sdk.rpc.Service -import com.google.protobuf.ByteString -import io.grpc.Channel -import io.grpc.ManagedChannelBuilder -import kotlinx.coroutines.* -import kotlinx.coroutines.Dispatchers.IO -import kotlinx.coroutines.channels.BroadcastChannel -import kotlinx.coroutines.channels.ConflatedBroadcastChannel -import kotlinx.coroutines.channels.ReceiveChannel -import kotlinx.coroutines.channels.distinct -import java.io.Closeable -import java.util.concurrent.TimeUnit - -/** - * Serves as a source of compact blocks received from the light wallet server. Once started, it will - * request all the appropriate blocks and then stream them into the channel returned when calling [start]. - */ -class CompactBlockStream private constructor() { - lateinit var connection: Connection - - // TODO: improve the creation of this channel (tweak its settings to use mobile device responsibly) and make sure it is properly cleaned up - constructor(host: String, port: Int) : this( - ManagedChannelBuilder.forAddress(host, port).usePlaintext().build() - ) - - constructor(channel: Channel) : this() { - connection = Connection(channel) - } - - fun start( - scope: CoroutineScope, - startingBlockHeight: Int = Int.MAX_VALUE, - batchSize: Int = DEFAULT_BATCH_SIZE, - pollFrequencyMillis: Long = DEFAULT_POLL_INTERVAL - ): ReceiveChannel { - if(connection.isClosed()) throw CompactBlockStreamException.ConnectionClosed - twig("starting") - scope.launch { - twig("preparing to stream blocks...") - delay(1000L) // TODO: we can probably get rid of this delay. - try { - connection.use { - twig("requesting latest block height") - var latestBlockHeight = it.getLatestBlockHeight() - twig("responded with latest block height of $latestBlockHeight") - if (startingBlockHeight < latestBlockHeight) { - twig("downloading missing blocks from $startingBlockHeight to $latestBlockHeight") - latestBlockHeight = it.downloadMissingBlocks(startingBlockHeight, batchSize) - twig("done downloading missing blocks") - } - it.streamBlocks(pollFrequencyMillis, latestBlockHeight) - } - } catch (t: Throwable) { - twig("throwing $t") - throw CompactBlockStreamException.FalseStart(t) - } - } - - return connection.subscribe() - } - - fun progress() = connection.progress().distinct() - - fun stop() { - twig("stopping") - connection.close() - } - - companion object { - const val DEFAULT_BATCH_SIZE = 10_000 - const val DEFAULT_POLL_INTERVAL = 75_000L - const val DEFAULT_RETRIES = 5 - } - - inner class Connection(private val channel: Channel): Closeable { - private var job: Job? = null - private var syncJob: Job? = null - private val compactBlockChannel = BroadcastChannel(100) - private val latestBlockHeightChannel = ConflatedBroadcastChannel() - private val progressChannel = ConflatedBroadcastChannel() - - fun createStub(timeoutMillis: Long = 60_000L): CompactTxStreamerBlockingStub { - return CompactTxStreamerGrpc.newBlockingStub(channel).withDeadlineAfter(timeoutMillis, TimeUnit.MILLISECONDS) - } - - fun subscribe() = compactBlockChannel.openSubscription() - - fun progress() = progressChannel.openSubscription() - - fun latestHeights() = latestBlockHeightChannel.openSubscription() - - /** - * Download all the missing blocks and return the height of the last block downloaded, which can be used to - * calculate the total number of blocks downloaded. - */ - suspend fun downloadMissingBlocks(startingBlockHeight: Int, batchSize: Int = DEFAULT_BATCH_SIZE): Int { - twig("downloadingMissingBlocks starting at $startingBlockHeight") - val latestBlockHeight = getLatestBlockHeight() - var downloadedBlockHeight = startingBlockHeight - // if blocks are missing then download them - if (startingBlockHeight < latestBlockHeight) { - val missingBlockCount = latestBlockHeight - startingBlockHeight + 1 - val batches = missingBlockCount / batchSize + (if (missingBlockCount.rem(batchSize) == 0) 0 else 1) - var progress: Int - twig("found $missingBlockCount missing blocks, downloading in $batches batches...") - for (i in 1..batches) { - retryUpTo(DEFAULT_RETRIES) { - twig("beginning batch $i") - val end = Math.min(startingBlockHeight + (i * batchSize), latestBlockHeight + 1) - loadBlockRange(downloadedBlockHeight..(end-1)) - progress = Math.round(i/batches.toFloat() * 100) - progressChannel.send(progress) - downloadedBlockHeight = end - twig("finished batch $i of $batches\n") - } - } -// progressChannel.cancel() - } else { - twig("no missing blocks to download!") - } - return downloadedBlockHeight - } - - suspend fun getLatestBlockHeight(): Int = withContext(IO) { - createStub().getLatestBlock(Service.ChainSpec.newBuilder().build()).height.toInt() - } - - suspend fun submitTransaction(raw: ByteArray) = withContext(IO) { - val request = Service.RawTransaction.newBuilder().setData(ByteString.copyFrom(raw)).build() - createStub().sendTransaction(request) - } - - suspend fun streamBlocks(pollFrequencyMillis: Long = DEFAULT_POLL_INTERVAL, startingBlockHeight: Int = Int.MAX_VALUE) = withContext(IO) { - twig("streamBlocks started at $startingBlockHeight with interval $pollFrequencyMillis") - progressChannel.send(100) // anytime we make it to this method, we're done catching up - // start with the next block, unless we were asked to start before then - var nextBlockHeight = Math.min(startingBlockHeight, getLatestBlockHeight() + 1) - while (isActive && !compactBlockChannel.isClosedForSend) { - retryUpTo(DEFAULT_RETRIES) { - twig("polling for next block in stream on thread ${Thread.currentThread().name} . . .") - val latestBlockHeight = getLatestBlockHeight() - if (latestBlockHeight >= nextBlockHeight) { - twig("found a new block! (latest: $latestBlockHeight) on thread ${Thread.currentThread().name}") - loadBlockRange(nextBlockHeight..latestBlockHeight) - nextBlockHeight = latestBlockHeight + 1 - } else { - twig("no new block yet (latest: $latestBlockHeight) on thread ${Thread.currentThread().name}") - } - twig("delaying $pollFrequencyMillis before polling for next block in stream") - delay(pollFrequencyMillis) - } - } - } - - private suspend fun retryUpTo(retries: Int, initialDelay:Int = 10, block: suspend () -> Unit) { - var failedAttempts = 0 - while (failedAttempts < retries) { - try { - block() - return - } catch (t: Throwable) { - failedAttempts++ - if (failedAttempts >= retries) throw t - val duration = Math.pow(initialDelay.toDouble(), failedAttempts.toDouble()).toLong() - twig("failed due to $t retrying (${failedAttempts+1}/$retries) in ${duration}s...") - delay(duration) - } - } - } - - suspend fun loadBlockRange(range: IntRange): Int = withContext(IO) { - twig("requesting block range $range on thread ${Thread.currentThread().name}") - val result = createStub(90_000L).getBlockRange(range.toBlockRange()) - twig("done requesting block range") - var resultCount = 0 - while (checkNextBlock(result)) { //calls result.hasNext, which blocks because we use a blockingStub - resultCount++ - val nextBlock = result.next() - twig("...while loading block range $range, received new block ${nextBlock.height} on thread ${Thread.currentThread().name}. Sending...") - compactBlockChannel.send(nextBlock) - latestBlockHeightChannel.send(nextBlock.height.toInt()) - twig("...done sending block ${nextBlock.height}") - } - twig("done loading block range $range") - resultCount - } - - /* this helper method is used to allow for logic (like logging) before blocking on the current thread */ - private fun checkNextBlock(result: MutableIterator): Boolean { - twig("awaiting next block...") - return result.hasNext() - } - - fun isClosed(): Boolean { - return compactBlockChannel.isClosedForSend - } - - override fun close() { - compactBlockChannel.cancel() - progressChannel.cancel() - syncJob?.cancel() - syncJob = null - job?.cancel() - job = null - } - } -} \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/data/MockSynchronizer.kt b/src/main/java/cash/z/wallet/sdk/data/MockSynchronizer.kt index b315e7ae..52666662 100644 --- a/src/main/java/cash/z/wallet/sdk/data/MockSynchronizer.kt +++ b/src/main/java/cash/z/wallet/sdk/data/MockSynchronizer.kt @@ -1,6 +1,9 @@ package cash.z.wallet.sdk.data +import cash.z.wallet.sdk.block.CompactBlockProcessor +import cash.z.wallet.sdk.block.ProcessorConfig import cash.z.wallet.sdk.dao.WalletTransaction +import cash.z.wallet.sdk.ext.MINERS_FEE_ZATOSHI import cash.z.wallet.sdk.secure.Wallet import kotlinx.coroutines.* import kotlinx.coroutines.channels.ConflatedBroadcastChannel @@ -24,7 +27,6 @@ import kotlin.random.nextLong * will send regular updates such that it reaches 100 in this amount of time. * @param activeTransactionUpdateFrequency the amount of time in milliseconds between updates to an active * transaction's state. Active transactions move through their lifecycle and increment their state at this rate. - * @param isFirstRun whether this Mock should return `true` for isFirstRun. Defaults to a random boolean. * @param isStale whether this Mock should return `true` for isStale. When null, this will follow the default behavior * of returning true about 10% of the time. * @param onSynchronizerErrorListener presently ignored because there are not yet any errors in mock. @@ -33,7 +35,6 @@ open class MockSynchronizer( private val transactionInterval: Long = 30_000L, private val initialLoadDuration: Long = 5_000L, private val activeTransactionUpdateFrequency: Long = 3_000L, - private val isFirstRun: Boolean = Random.nextBoolean(), private var isStale: Boolean? = null, override var onSynchronizerErrorListener: ((Throwable?) -> Boolean)? = null // presently ignored (there are no errors in mock yet) ) : Synchronizer, CoroutineScope { @@ -96,14 +97,6 @@ open class MockSynchronizer( return result } - /** - * Returns [isFirstRun] as provided during initialization of this MockSynchronizer. - */ - override suspend fun isFirstRun(): Boolean { - twig("checking isFirstRun: $isFirstRun") - return isFirstRun - } - /** * Returns the [mockAddress]. This address is not usable. */ @@ -116,7 +109,7 @@ open class MockSynchronizer( if (transactions.size != 0) { return transactions.fold(0L) { acc, tx -> if (tx.isSend && tx.isMined) acc - tx.value else acc + tx.value - } - 10_000L // miner's fee + } - MINERS_FEE_ZATOSHI } return 0L } @@ -262,7 +255,7 @@ open class MockSynchronizer( if (tx.isSend && tx.isMined) acc - tx.value else acc + tx.value } } - balanceChannel.send(Wallet.WalletBalance(balance, balance - 10000 /* miner's fee */)) + balanceChannel.send(Wallet.WalletBalance(balance, balance - MINERS_FEE_ZATOSHI)) } // other collaborators add to the list, periodically. This simulates, real-world, non-distinct updates. delay(Random.nextLong(transactionInterval / 2)) diff --git a/src/main/java/cash/z/wallet/sdk/data/SdkSynchronizer.kt b/src/main/java/cash/z/wallet/sdk/data/SdkSynchronizer.kt index 980ed2a0..97c6fca0 100644 --- a/src/main/java/cash/z/wallet/sdk/data/SdkSynchronizer.kt +++ b/src/main/java/cash/z/wallet/sdk/data/SdkSynchronizer.kt @@ -1,8 +1,9 @@ package cash.z.wallet.sdk.data +import cash.z.wallet.sdk.block.CompactBlockProcessor import cash.z.wallet.sdk.dao.WalletTransaction -import cash.z.wallet.sdk.data.SdkSynchronizer.SyncState.* import cash.z.wallet.sdk.exception.SynchronizerException +import cash.z.wallet.sdk.exception.WalletException import cash.z.wallet.sdk.secure.Wallet import kotlinx.coroutines.* import kotlinx.coroutines.Dispatchers.IO @@ -18,7 +19,6 @@ import kotlin.coroutines.CoroutineContext * Another way of thinking about this class is the reference that demonstrates how all the pieces can be tied * together. * - * @param downloader the component that downloads compact blocks and exposes them as a stream * @param processor the component that saves the downloaded compact blocks to the cache and then scans those blocks for * data related to this wallet. * @param repository the component that exposes streams of wallet transaction information. @@ -31,14 +31,11 @@ import kotlin.coroutines.CoroutineContext * number represents the number of milliseconds the synchronizer will wait before checking for newly mined blocks. */ class SdkSynchronizer( - private val downloader: CompactBlockStream, private val processor: CompactBlockProcessor, private val repository: TransactionRepository, private val activeTransactionManager: ActiveTransactionManager, private val wallet: Wallet, - private val batchSize: Int = 1000, - private val staleTolerance: Int = 10, - private val blockPollFrequency: Long = CompactBlockStream.DEFAULT_POLL_INTERVAL + private val staleTolerance: Int = 10 ) : Synchronizer { /** @@ -103,7 +100,13 @@ class SdkSynchronizer( failure = null blockJob = parentScope.launch(CoroutineExceptionHandler(exceptionHandler)) { supervisorScope { - continueWithState(determineState()) + try { + wallet.initialize() + } catch (e: WalletException.AlreadyInitializedException) { + twig("Warning: wallet already initialized but this is safe to ignore " + + "because the SDK now automatically detects where to start downloading.") + } + onReady() } } return this @@ -116,7 +119,6 @@ class SdkSynchronizer( */ override fun stop() { twig("stopping") - downloader.stop().also { twig("downloader stopped") } repository.stop().also { twig("repository stopped") } activeTransactionManager.stop().also { twig("activeTransactionManager stopped") } // TODO: investigate whether this is necessary and remove or improve, accordingly @@ -146,7 +148,7 @@ class SdkSynchronizer( * switches from catching up on missed blocks to periodically monitoring for newly mined blocks. */ override fun progress(): ReceiveChannel { - return downloader.progress() + return processor.progress() } /** @@ -170,27 +172,15 @@ class SdkSynchronizer( * @return true when the local data is significantly out of sync with the remote server and the app data is stale. */ override suspend fun isStale(): Boolean = withContext(IO) { - val latestBlockHeight = downloader.connection.getLatestBlockHeight() - val ourHeight = processor.cacheDao.latestBlockHeight() - val tolerance = 10 + val latestBlockHeight = processor.downloader.getLatestBlockHeight() + val ourHeight = processor.downloader.getLastDownloadedHeight() + val tolerance = staleTolerance val delta = latestBlockHeight - ourHeight twig("checking whether out of sync. " + "LatestHeight: $latestBlockHeight ourHeight: $ourHeight Delta: $delta tolerance: $tolerance") delta > tolerance } - /** - * A flag to indicate that the initial state of this synchronizer was firstRun. This is useful for knowing whether - * initializing the database is required and whether to show things like"first run walk-throughs." - * - * @return true when this synchronizer has not been run before on this device or when cache has been cleared since - * the last run. - */ - override suspend fun isFirstRun(): Boolean = withContext(IO) { - initialState is FirstRun - } - - /* Operations */ /** @@ -234,80 +224,17 @@ class SdkSynchronizer( // Private API // - /** - * After determining the initial state, continue based on those findings. - * - * @param syncState the sync state found - */ - private fun CoroutineScope.continueWithState(syncState: SyncState): Job { - return when (syncState) { - FirstRun -> onFirstRun() - is CacheOnly -> onCacheOnly(syncState) - is ReadyToProcess -> onReady(syncState) - } - } - - /** - * Logic for the first run. This is when the wallet gets initialized, which includes setting up the dataDB and - * preloading it with data corresponding to the wallet birthday. - */ - private fun CoroutineScope.onFirstRun(): Job { - twig("this appears to be a fresh install, beginning first run of application") - val firstRunStartHeight = wallet.initialize() // should get the latest sapling tree and return that height - twig("wallet firstRun returned a value of $firstRunStartHeight") - return continueWithState(ReadyToProcess(firstRunStartHeight)) - } - - /** - * Logic for starting the Synchronizer when no scans have yet occurred. Takes care of initializing the dataDb and - * then - */ - private fun CoroutineScope.onCacheOnly(syncState: CacheOnly): Job { - twig("we have cached blocks but no data DB, beginning pre-cached version of application") - val firstRunStartHeight = wallet.initialize(syncState.startingBlockHeight) - twig("wallet has already cached up to a height of $firstRunStartHeight") - return continueWithState(ReadyToProcess(firstRunStartHeight)) - } /** * Logic for starting the Synchronizer once it is ready for processing. All starts eventually end with this method. */ - private fun CoroutineScope.onReady(syncState: ReadyToProcess) = launch { - twig("synchronization is ready to begin at height ${syncState.startingBlockHeight}") - // TODO: for PIR concerns, introduce some jitter here for where, exactly, the downloader starts - val blockChannel = - downloader.start( - this, - syncState.startingBlockHeight, - batchSize, - pollFrequencyMillis = blockPollFrequency - ) - launch { monitorProgress(downloader.progress()) } + private fun CoroutineScope.onReady() = launch { + twig("synchronization is ready to begin!") launch { monitorTransactions(repository.allTransactions().distinct()) } + activeTransactionManager.start() repository.start(this) - processor.processBlocks(blockChannel) - } - - /** - * Monitor download progress in order to trigger a scan the moment all blocks have been received. This reduces the - * amount of time it takes to get accurate balance information since scan intervals are fairly long. - */ - private suspend fun monitorProgress(progressChannel: ReceiveChannel) = withContext(IO) { - twig("beginning to monitor download progress") - for (i in progressChannel) { - if(i >= 100) { - twig("triggering a proactive scan in a second because all missing blocks have been loaded") - delay(1000L) - launch { - twig("triggering proactive scan!") - processor.scanBlocks() - twig("done triggering proactive scan!") - } - break - } - } - twig("done monitoring download progress") + processor.start() } /** @@ -324,32 +251,6 @@ class SdkSynchronizer( twig("done monitoring transactions in order to update the balance") } - /** - * Determine the initial state of the data by checking whether the dataDB is initialized and the last scanned block - * height. This is considered a first run if no blocks have been processed. - */ - private suspend fun determineState(): SyncState = withContext(IO) { - twig("determining state (has the app run before, what block did we last see, etc.)") - initialState = if (processor.dataDbExists) { - val isInitialized = repository.isInitialized() - // this call blocks because it does IO - val startingBlockHeight = Math.max(processor.lastProcessedBlock(), repository.lastScannedHeight()) - - twig("cacheDb exists with last height of $startingBlockHeight and isInitialized = $isInitialized") - if (!repository.isInitialized()) FirstRun else ReadyToProcess(startingBlockHeight) - } else if(processor.cachDbExists) { - // this call blocks because it does IO - val startingBlockHeight = processor.lastProcessedBlock() - twig("cacheDb exists with last height of $startingBlockHeight") - if (startingBlockHeight <= 0) FirstRun else CacheOnly(startingBlockHeight) - } else { - FirstRun - } - - twig("determined ${initialState::class.java.simpleName}") - initialState - } - /** * Wraps exceptions, logs them and then invokes the [onSynchronizerErrorListener], if it exists. */ diff --git a/src/main/java/cash/z/wallet/sdk/data/Synchronizer.kt b/src/main/java/cash/z/wallet/sdk/data/Synchronizer.kt index 4826b264..534c68af 100644 --- a/src/main/java/cash/z/wallet/sdk/data/Synchronizer.kt +++ b/src/main/java/cash/z/wallet/sdk/data/Synchronizer.kt @@ -66,15 +66,6 @@ interface Synchronizer { */ suspend fun isStale(): Boolean - /** - * A flag to indicate that this is the first run of this Synchronizer on this device. This is useful for knowing - * whether to initialize databases or other required resources, as well as whether to show walk-throughs. - * - * @return true when this is the first run. Implementations can set criteria for that but typically it will be when - * the database needs to be initialized. - */ - suspend fun isFirstRun(): Boolean - /** * Gets or sets a global error listener. This is a useful hook for handling unexpected critical errors. * diff --git a/src/main/java/cash/z/wallet/sdk/service/LightWalletGrpcService.kt b/src/main/java/cash/z/wallet/sdk/service/LightWalletGrpcService.kt new file mode 100644 index 00000000..68d4810f --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/service/LightWalletGrpcService.kt @@ -0,0 +1,56 @@ +package cash.z.wallet.sdk.service + +import cash.z.wallet.sdk.entity.CompactBlock +import cash.z.wallet.sdk.ext.toBlockHeight +import cash.z.wallet.sdk.rpc.CompactFormats +import cash.z.wallet.sdk.rpc.CompactTxStreamerGrpc +import cash.z.wallet.sdk.rpc.Service +import com.google.protobuf.ByteString +import io.grpc.Channel +import io.grpc.ManagedChannelBuilder +import java.util.concurrent.TimeUnit + +class LightWalletGrpcService(private val channel: Channel) : LightWalletService { + + constructor(host: String, port: Int = 9067) : this(ManagedChannelBuilder.forAddress(host, port).usePlaintext().build()) + + /* LightWalletService implementation */ + + override fun getBlockRange(heightRange: IntRange): List { + return channel.createStub(90L).getBlockRange(heightRange.toBlockRange()).toList() + } + + override fun getLatestBlockHeight(): Int { + return channel.createStub(10L).getLatestBlock(Service.ChainSpec.newBuilder().build()).height.toInt() + } + + override fun submitTransaction(raw: ByteArray): Service.SendResponse { + val request = Service.RawTransaction.newBuilder().setData(ByteString.copyFrom(raw)).build() + return channel.createStub().sendTransaction(request) + } + + + // + // Utilities + // + + private fun Channel.createStub(timeoutSec: Long = 60L): CompactTxStreamerGrpc.CompactTxStreamerBlockingStub = + CompactTxStreamerGrpc + .newBlockingStub(this) + .withDeadlineAfter(timeoutSec, TimeUnit.SECONDS) + + private fun IntRange.toBlockRange(): Service.BlockRange = + Service.BlockRange.newBuilder() + .setStart(this.first.toBlockHeight()) + .setEnd(this.last.toBlockHeight()) + .build() + + private fun Iterator.toList(): List = + mutableListOf().apply { + while (hasNext()) { + val compactBlock = next() + this@apply += CompactBlock(compactBlock.height.toInt(), compactBlock.toByteArray()) + } + } +} + diff --git a/src/main/java/cash/z/wallet/sdk/service/LightWalletService.kt b/src/main/java/cash/z/wallet/sdk/service/LightWalletService.kt new file mode 100644 index 00000000..da705295 --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/service/LightWalletService.kt @@ -0,0 +1,28 @@ +package cash.z.wallet.sdk.service + +import cash.z.wallet.sdk.entity.CompactBlock +import cash.z.wallet.sdk.rpc.Service + +/** + * Service for interacting with lightwalletd. Implementers of this service should make blocking calls because + * async concerns are handled at a higher level. + */ +interface LightWalletService { + /** + * Return the given range of blocks. + * + * @param heightRange the inclusive range to fetch. For instance if 1..5 is given, then every block in that range + * will be fetched, including 1 and 5. + */ + fun getBlockRange(heightRange: IntRange): List + + /** + * Return the latest block height known to the service. + */ + fun getLatestBlockHeight(): Int + + /** + * Submit a raw transaction. + */ + fun submitTransaction(transactionRaw: ByteArray): Service.SendResponse +} \ No newline at end of file diff --git a/src/test/java/cash/z/wallet/sdk/block/CompactBlockProcessorTest.kt b/src/test/java/cash/z/wallet/sdk/block/CompactBlockProcessorTest.kt new file mode 100644 index 00000000..74425494 --- /dev/null +++ b/src/test/java/cash/z/wallet/sdk/block/CompactBlockProcessorTest.kt @@ -0,0 +1,153 @@ +package cash.z.wallet.sdk.block + +import cash.z.wallet.sdk.data.TransactionRepository +import cash.z.wallet.sdk.data.TroubleshootingTwig +import cash.z.wallet.sdk.data.Twig +import cash.z.wallet.sdk.entity.CompactBlock +import cash.z.wallet.sdk.ext.SAPLING_ACTIVATION_HEIGHT +import cash.z.wallet.sdk.jni.RustBackendWelding +import cash.z.wallet.sdk.service.LightWalletService +import com.nhaarman.mockitokotlin2.* +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.junit.jupiter.MockitoSettings +import org.mockito.quality.Strictness + +@ExtendWith(MockitoExtension::class) +@MockitoSettings(strictness = Strictness.LENIENT) +internal class CompactBlockProcessorTest { + + private val frequency = 5L + + // Mocks/Spys + @Mock lateinit var rustBackend: RustBackendWelding + lateinit var processor: CompactBlockProcessor + + // Test variables + private var latestBlockHeight: Int = 500_000 + private var lastDownloadedHeight: Int = SAPLING_ACTIVATION_HEIGHT + private var lastScannedHeight: Int = SAPLING_ACTIVATION_HEIGHT + private var errorBlock: Int = -1 + + @BeforeEach + fun setUp( + @Mock lightwalletService: LightWalletService, + @Mock compactBlockStore: CompactBlockStore, + @Mock repository: TransactionRepository + ) { + Twig.plant(TroubleshootingTwig()) + + + lightwalletService.stub { + onBlocking { + getBlockRange(any()) + }.thenAnswer { invocation -> + val range = invocation.arguments[0] as IntRange + range.map { CompactBlock(it, ByteArray(0)) } + } + } + lightwalletService.stub { + onBlocking { + getLatestBlockHeight() + }.thenAnswer { latestBlockHeight } + } + + compactBlockStore.stub { + onBlocking { + write(any()) + }.thenAnswer { invocation -> + val lastBlockHeight = (invocation.arguments[0] as List).last().height + lastDownloadedHeight = lastBlockHeight + Unit + } + } + compactBlockStore.stub { + onBlocking { + getLatestHeight() + }.thenAnswer { lastDownloadedHeight } + } + compactBlockStore.stub { + onBlocking { + rewindTo(any()) + }.thenAnswer { invocation -> + lastDownloadedHeight = invocation.arguments[0] as Int + Unit + } + } + repository.stub { + onBlocking { + lastScannedHeight() + }.thenAnswer { lastScannedHeight } + } + + val config = ProcessorConfig(retries = 1, blockPollFrequencyMillis = frequency, downloadBatchSize = 50_000) + val downloader = spy(CompactBlockDownloader(lightwalletService, compactBlockStore)) + processor = spy(CompactBlockProcessor(config, downloader, repository, rustBackend)) + + whenever(rustBackend.validateCombinedChain(any(), any())).thenAnswer { + errorBlock + } + + whenever(rustBackend.scanBlocks(any(), any())).thenAnswer { + true + } + } + + @AfterEach + fun tearDown() { + } + + @Test + fun `check for OBOE when downloading`() = runBlocking { + // if the last block downloaded was 350_000, then we already have that block and should start with 350_001 + lastDownloadedHeight = 350_000 + + processBlocks() + verify(processor).downloadNewBlocks(350_001..latestBlockHeight) + } + + @Test + fun `chain error rewinds by expected amount`() = runBlocking { + // if the highest block whose prevHash doesn't match happens at block 300_010 + errorBlock = 300_010 + + // then we should rewind the default (10) blocks + val expectedBlock = errorBlock - processor.config.rewindDistance + processBlocks(100L) + verify(processor.downloader, atLeastOnce()).rewindTo(expectedBlock) + verify(rustBackend, atLeastOnce()).rewindToHeight("", expectedBlock) + assertNotNull(processor) + } + + @Test + fun `chain error downloads expected number of blocks`() = runBlocking { + // if the highest block whose prevHash doesn't match happens at block 300_010 + // and our rewind distance is the default (10), then we want to download exactly ten blocks + errorBlock = 300_010 + + // plus 1 because the range is inclusive + val expectedRange = (errorBlock - processor.config.rewindDistance + 1)..latestBlockHeight + processBlocks(1500L) + verify(processor, atLeastOnce()).downloadNewBlocks(expectedRange) + } + + private fun processBlocks(delayMillis: Long? = null) = runBlocking { + launch { processor.start() } + val progressChannel = processor.progress() + for (i in progressChannel) { + if(i >= 100) { + if(delayMillis != null) delay(delayMillis) + processor.stop() + break + } + } + } +} \ No newline at end of file diff --git a/src/test/java/cash/z/wallet/sdk/data/CompactBlockDownloaderTest.kt b/src/test/java/cash/z/wallet/sdk/data/CompactBlockDownloaderTest.kt deleted file mode 100644 index 5f571b1f..00000000 --- a/src/test/java/cash/z/wallet/sdk/data/CompactBlockDownloaderTest.kt +++ /dev/null @@ -1,173 +0,0 @@ -package cash.z.wallet.sdk.data - -import cash.z.wallet.anyNotNull -import cash.z.wallet.sdk.ext.toBlockHeight -import cash.z.wallet.sdk.rpc.CompactFormats -import cash.z.wallet.sdk.rpc.CompactTxStreamerGrpc.CompactTxStreamerBlockingStub -import cash.z.wallet.sdk.rpc.Service -import com.nhaarman.mockitokotlin2.* -import kotlinx.coroutines.* -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.mockito.ArgumentMatchers.any -import org.mockito.Mock -import org.mockito.junit.jupiter.MockitoExtension -import org.mockito.junit.jupiter.MockitoSettings -import org.mockito.quality.Strictness -import kotlin.system.measureTimeMillis -import org.junit.Rule -import io.grpc.testing.GrpcServerRule -import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport - - -@ExtendWith(MockitoExtension::class) -@MockitoSettings(strictness = Strictness.LENIENT) // allows us to setup the blockingStub once, with everything, rather than using custom stubs for each test -@EnableRuleMigrationSupport -class CompactBlockDownloaderTest { - - lateinit var downloader: CompactBlockStream - lateinit var connection: CompactBlockStream.Connection - val job = Job() - val io = CoroutineScope(Dispatchers.IO + job) - - @Rule - var grpcServerRule = GrpcServerRule() - - @BeforeEach - fun setUp(@Mock blockingStub: CompactTxStreamerBlockingStub) { - whenever(blockingStub.getLatestBlock(any())).doAnswer { - getLatestBlock() - } - // when asked for a block range, create an array of blocks and return an iterator over them with a slight delay between iterations - whenever(blockingStub.getBlockRange(any())).doAnswer { - val serviceRange = it.arguments[0] as Service.BlockRange - val range = serviceRange.start.height..serviceRange.end.height - val blocks = mutableListOf() - System.err.println("[Mock Connection] creating blocks in range: $range") - for (i in range) { - blocks.add(CompactFormats.CompactBlock.newBuilder().setHeight(i).build()) - } - val blockIterator = blocks.iterator() - - val delayedIterator = object : Iterator { - override fun hasNext() = blockIterator.hasNext() - - override fun next(): CompactFormats.CompactBlock { - Thread.sleep(10L) - return blockIterator.next() - } - } - delayedIterator - } - downloader = CompactBlockStream(grpcServerRule.channel, TroubleshootingTwig()) - connection = spy(downloader.connection) - whenever(connection.createStub(anyNotNull())).thenReturn(blockingStub) - } - - @AfterEach - fun tearDown() { - downloader.stop() - io.cancel() - } - - @Test - fun `mock configuration sanity check`() = runBlocking { - assertEquals(getLatestBlock().height, connection.getLatestBlockHeight(), "Unexpected height. Verify that mocks are properly configured.") - } - - @Test - fun `downloading missing blocks happens in chunks`() = runBlocking { - val start = getLatestBlock().height.toInt() - 31 - val downloadCount = connection.downloadMissingBlocks(start, 10) - start - assertEquals(32, downloadCount) - -// verify(connection).getLatestBlockHeight() -// verify(connection).loadBlockRange(start..(start + 9)) // a range of 10 block is requested -// verify(connection, times(4)).loadBlockRange(anyNotNull()) // 4 batches are required - } - - @Test - fun `channel contains expected blocks`() = runBlocking { - val mailbox = connection.subscribe() - var blockCount = 0 - val start = getLatestBlock().height - 31L - io.launch { - connection.downloadMissingBlocks(start.toInt(), 10) - mailbox.cancel() // exits the for loop, below, once downloading is complete - } - for(block in mailbox) { - println("got block with height ${block.height} on thread ${Thread.currentThread().name}") - blockCount++ - } - assertEquals(32, blockCount) - } - - // lots of logging here because this is more of a sanity test for peace of mind - @Test - fun `streaming yields the latest blocks with proper timing`() = runBlocking { - // just tweak these a bit for sanity rather than making a bunch of tests that would be slow - val pollInterval = BLOCK_INTERVAL_MILLIS/2L - val repetitions = 3 - - println("${System.currentTimeMillis()} : starting with blockInterval $BLOCK_INTERVAL_MILLIS and pollInterval $pollInterval") - val mailbox = connection.subscribe() - io.launch { - connection.streamBlocks(pollInterval) - } - // sync up with the block interval, first - mailbox.receive() - - // now, get a few blocks and measure the expected time - val deltaTime = measureTimeMillis { - repeat(repetitions) { - println("${System.currentTimeMillis()} : checking the mailbox on thread ${Thread.currentThread().name}...") - val mail = mailbox.receive() - println("${System.currentTimeMillis()} : ...got ${mail.height} in the mail! on thread ${Thread.currentThread().name}") - } - } - val totalIntervals = repetitions * BLOCK_INTERVAL_MILLIS - val bounds = (totalIntervals - pollInterval)..(totalIntervals + pollInterval) - println("${System.currentTimeMillis()} : finished in $deltaTime and it was between $bounds") - - mailbox.cancel() - assertTrue(bounds.contains(deltaTime), "Blocks received ${if(bounds.first < deltaTime) "slower" else "faster"} than expected. $deltaTime should be in the range of $bounds") - } - - @Test - fun `downloader gets missing blocks and then streams`() = runBlocking { - val targetHeight = getLatestBlock().height.toInt() + 3 - val initialBlockHeight = targetHeight - 30 - println("starting from $initialBlockHeight to $targetHeight") - val mailbox = downloader.start(io, initialBlockHeight, 10, 500L) - - // receive from channel until we reach the target height, counting blocks along the way - var firstBlock: CompactFormats.CompactBlock? = null - var blockCount = 0 - do { - println("waiting for block number $blockCount...") - val block = mailbox.receive() - println("...received block ${block.height} on thread ${Thread.currentThread().name}") - blockCount++ - if (firstBlock == null) firstBlock = block - } while (block.height < targetHeight) - - mailbox.cancel() - assertEquals(firstBlock?.height, initialBlockHeight, "Failed to start at block $initialBlockHeight") - assertEquals(targetHeight - initialBlockHeight + 1L, blockCount.toLong(), "Incorrect number of blocks, verify that there are no duplicates in the test output") - } - - companion object { - const val BLOCK_INTERVAL_MILLIS = 1000L - - private fun getLatestBlock(): Service.BlockID { - // number of intervals that have passed (without rounding...) - val intervalCount = System.currentTimeMillis() / BLOCK_INTERVAL_MILLIS - return intervalCount.toInt().toBlockHeight() - } - } - -} \ No newline at end of file From a78c98b8aa0a00a2e8cc261370dddf114b6ba5fb Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:25:37 -0400 Subject: [PATCH 04/18] Add more support for memos. --- .../cash/z/wallet/sdk/dao/TransactionDao.kt | 17 +++++++++++++-- .../sdk/data/PollingTransactionRepository.kt | 21 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/main/java/cash/z/wallet/sdk/dao/TransactionDao.kt b/src/main/java/cash/z/wallet/sdk/dao/TransactionDao.kt index b078350f..45f07fc6 100644 --- a/src/main/java/cash/z/wallet/sdk/dao/TransactionDao.kt +++ b/src/main/java/cash/z/wallet/sdk/dao/TransactionDao.kt @@ -30,7 +30,15 @@ interface TransactionDao { CASE WHEN transactions.raw IS NOT NULL THEN sent_notes.value ELSE received_notes.value - END AS value + END AS value, + CASE + WHEN transactions.raw IS NOT NULL THEN sent_notes.memo IS NOT NULL + ELSE received_notes.memo IS NOT NULL + END AS rawMemoExists, + CASE + WHEN transactions.raw IS NOT NULL THEN sent_notes.id_note + ELSE received_notes.id_note + END AS noteId FROM transactions LEFT JOIN sent_notes ON transactions.id_tx = sent_notes.tx @@ -45,11 +53,16 @@ interface TransactionDao { } data class WalletTransaction( + val noteId: Long = 0L, val txId: Long = 0L, val value: Long = 0L, val height: Int? = null, val isSend: Boolean = false, val timeInSeconds: Long = 0L, val address: String? = null, - val isMined: Boolean = false + val isMined: Boolean = false, + // does the raw transaction contain a memo? + val rawMemoExists: Boolean = false, + // TODO: investigate populating this with SQL rather than a separate SDK call. then get rid of rawMemoExists. + var memo: String? = null ) \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/data/PollingTransactionRepository.kt b/src/main/java/cash/z/wallet/sdk/data/PollingTransactionRepository.kt index d972bf61..7bc839df 100644 --- a/src/main/java/cash/z/wallet/sdk/data/PollingTransactionRepository.kt +++ b/src/main/java/cash/z/wallet/sdk/data/PollingTransactionRepository.kt @@ -9,6 +9,7 @@ import cash.z.wallet.sdk.dao.WalletTransaction import cash.z.wallet.sdk.db.DerivedDataDb import cash.z.wallet.sdk.entity.Transaction import cash.z.wallet.sdk.exception.RepositoryException +import cash.z.wallet.sdk.jni.RustBackendWelding import kotlinx.coroutines.* import kotlinx.coroutines.Dispatchers.IO import kotlinx.coroutines.channels.ConflatedBroadcastChannel @@ -20,7 +21,9 @@ import kotlinx.coroutines.channels.ReceiveChannel * changes. */ open class PollingTransactionRepository( + private val dataDbPath: String, private val derivedDataDb: DerivedDataDb, + private val rustBackend: RustBackendWelding, private val pollFrequencyMillis: Long = 2000L ) : TransactionRepository { @@ -30,12 +33,15 @@ open class PollingTransactionRepository( constructor( context: Context, dataDbName: String, + rustBackend: RustBackendWelding, pollFrequencyMillis: Long = 2000L, dbCallback: (DerivedDataDb) -> Unit = {} ) : this( + context.getDatabasePath(dataDbName).absolutePath, Room.databaseBuilder(context, DerivedDataDb::class.java, dataDbName) .setJournalMode(RoomDatabase.JournalMode.TRUNCATE) .build(), + rustBackend, pollFrequencyMillis ) { dbCallback(derivedDataDb) @@ -99,7 +105,7 @@ open class PollingTransactionRepository( if (hasChanged(previousTransactions, newTransactions)) { twig("loaded ${newTransactions.count()} transactions and changes were detected!") - allTransactionsChannel.send(newTransactions) + allTransactionsChannel.send(addMemos(newTransactions)) previousTransactions = newTransactions } else { twig("loaded ${newTransactions.count()} transactions but no changes detected.") @@ -110,6 +116,19 @@ open class PollingTransactionRepository( twig("Done polling for transactions") } + private suspend fun addMemos(newTransactions: List): List = withContext(IO){ + for (tx in newTransactions) { + if (tx.rawMemoExists) { + tx.memo = if(tx.isSend) { + rustBackend.getSentMemoAsUtf8(dataDbPath, tx.noteId) + } else { + rustBackend.getReceivedMemoAsUtf8(dataDbPath, tx.noteId) + } + } + } + newTransactions + } + private fun hasChanged(oldTxs: List?, newTxs: List): Boolean { fun pr(t: List?): String { From 94e2af728713bbf8252a01edb3483e64431b38c6 Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:26:53 -0400 Subject: [PATCH 05/18] Improve error handling and troubleshooting by surfacing more errors. --- .../sdk/data/ActiveTransactionManager.kt | 21 ++++++--- .../cash/z/wallet/sdk/exception/Exceptions.kt | 10 +++++ .../cash/z/wallet/sdk/ext/WalletService.kt | 29 +++++++++--- .../java/cash/z/wallet/sdk/secure/Wallet.kt | 44 ++++++++++++++----- 4 files changed, 82 insertions(+), 22 deletions(-) diff --git a/src/main/java/cash/z/wallet/sdk/data/ActiveTransactionManager.kt b/src/main/java/cash/z/wallet/sdk/data/ActiveTransactionManager.kt index 5694974a..4c0d145e 100644 --- a/src/main/java/cash/z/wallet/sdk/data/ActiveTransactionManager.kt +++ b/src/main/java/cash/z/wallet/sdk/data/ActiveTransactionManager.kt @@ -9,9 +9,12 @@ import kotlinx.coroutines.channels.ReceiveChannel import java.util.* import kotlin.coroutines.CoroutineContext import cash.z.wallet.sdk.data.TransactionState.* -import cash.z.wallet.sdk.rpc.CompactFormats +//import cash.z.wallet.sdk.rpc.CompactFormats +import cash.z.wallet.sdk.service.LightWalletService import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract /** * Manages active send/receive transactions. These are transactions that have been initiated but not completed with @@ -19,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong */ class ActiveTransactionManager( private val repository: TransactionRepository, - private val service: CompactBlockStream.Connection, + private val service: LightWalletService, private val wallet: Wallet ) : CoroutineScope { @@ -194,7 +197,15 @@ class ActiveTransactionManager( suspend fun sendToAddress(zatoshi: Long, toAddress: String, memo: String = "", fromAccountId: Int = 0) = withContext(Dispatchers.IO) { twig("creating send transaction for zatoshi value $zatoshi") val activeSendTransaction = create(zatoshi, toAddress.masked()) - val transactionId: Long = wallet.createRawSendTransaction(zatoshi, toAddress, memo, fromAccountId) // this call takes up to 20 seconds + val transactionId: Long = try { + // this call takes up to 20 seconds + wallet.createRawSendTransaction(zatoshi, toAddress, memo, fromAccountId) + } catch (t: Throwable) { + val reason = "${t.message}" + twig("Failed to create transaction due to: $reason") + failure(activeSendTransaction, reason) + return@withContext + } // cancellation basically just prevents sending to the network but we cannot cancel after this moment // well, technically we could still allow cancellation in the split second between this line of code and the upload request but lets not complicate things @@ -205,7 +216,7 @@ class ActiveTransactionManager( } if (transactionId < 0) { - failure(activeSendTransaction, "Failed to create, possibly due to insufficient funds or an invalid key") + failure(activeSendTransaction, "Failed to create, for unknown reason") return@withContext } val transactionRaw: ByteArray? = repository.findTransactionById(transactionId)?.raw @@ -308,4 +319,4 @@ sealed class TransactionState(val order: Int) { override fun toString(): String { return javaClass.simpleName } -} \ No newline at end of file +} diff --git a/src/main/java/cash/z/wallet/sdk/exception/Exceptions.kt b/src/main/java/cash/z/wallet/sdk/exception/Exceptions.kt index 112c4d16..8e048a33 100644 --- a/src/main/java/cash/z/wallet/sdk/exception/Exceptions.kt +++ b/src/main/java/cash/z/wallet/sdk/exception/Exceptions.kt @@ -1,5 +1,6 @@ package cash.z.wallet.sdk.exception +import java.lang.Exception import java.lang.RuntimeException /** @@ -25,6 +26,12 @@ sealed class SynchronizerException(message: String, cause: Throwable? = null) : sealed class CompactBlockProcessorException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) { class DataDbMissing(path: String): CompactBlockProcessorException("No data db file found at path $path. Verify " + "that the data DB has been initialized via `rustBackend.initDataDb(path)`") + open class ConfigurationException(message: String, cause: Throwable?) : CompactBlockProcessorException(message, cause) + class FileInsteadOfPath(fileName: String) : ConfigurationException("Invalid Path: the given path appears to be a" + + " file name instead of a path: $fileName. The RustBackend expects the absolutePath to the database rather" + + " than just the database filename because Rust does not access the app Context." + + " So pass in context.getDatabasePath(dbFileName).absolutePath instead of just dbFileName alone.", null) + class FailedReorgRepair(message: String) : CompactBlockProcessorException(message) } sealed class CompactBlockStreamException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) { @@ -45,4 +52,7 @@ sealed class WalletException(message: String, cause: Throwable? = null) : Runtim "Failed to parse file $directory/$file verify that it is formatted as #####.json, " + "where the first portion is an Int representing the height of the tree contained in the file" ) + class AlreadyInitializedException(cause: Throwable) : WalletException("Failed to initialize the blocks table" + + " because it already exists.", cause) + class FalseStart(cause: Throwable?) : WalletException("Failed to initialize wallet due to: $cause", cause) } \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/ext/WalletService.kt b/src/main/java/cash/z/wallet/sdk/ext/WalletService.kt index 9a1d24bc..02ce40b4 100644 --- a/src/main/java/cash/z/wallet/sdk/ext/WalletService.kt +++ b/src/main/java/cash/z/wallet/sdk/ext/WalletService.kt @@ -1,10 +1,29 @@ package cash.z.wallet.sdk.ext +import android.content.Context +import cash.z.wallet.sdk.data.twig import cash.z.wallet.sdk.rpc.Service +import kotlinx.coroutines.delay +import java.io.File inline fun Int.toBlockHeight(): Service.BlockID = Service.BlockID.newBuilder().setHeight(this.toLong()).build() -inline fun IntRange.toBlockRange(): Service.BlockRange = - Service.BlockRange.newBuilder() - .setStart(this.first.toBlockHeight()) - .setEnd(this.last.toBlockHeight()) - .build() + +suspend inline fun retryUpTo(retries: Int, initialDelay: Int = 10, block: () -> Unit) { + var failedAttempts = 0 + while (failedAttempts < retries) { + try { + block() + return + } catch (t: Throwable) { + failedAttempts++ + if (failedAttempts >= retries) throw t + val duration = Math.pow(initialDelay.toDouble(), failedAttempts.toDouble()).toLong() + twig("failed due to $t retrying (${failedAttempts + 1}/$retries) in ${duration}s...") + delay(duration) + } + } +} + +internal fun dbExists(appContext: Context, dbFileName: String): Boolean { + return File(appContext.getDatabasePath(dbFileName).absolutePath).exists() +} \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/secure/Wallet.kt b/src/main/java/cash/z/wallet/sdk/secure/Wallet.kt index 8c0bef5f..46caf4b8 100644 --- a/src/main/java/cash/z/wallet/sdk/secure/Wallet.kt +++ b/src/main/java/cash/z/wallet/sdk/secure/Wallet.kt @@ -2,11 +2,11 @@ package cash.z.wallet.sdk.secure import android.content.Context import cash.z.wallet.sdk.data.Bush -import cash.z.wallet.sdk.data.CompactBlockProcessor.Companion.SAPLING_ACTIVATION_HEIGHT import cash.z.wallet.sdk.data.twig import cash.z.wallet.sdk.data.twigTask import cash.z.wallet.sdk.exception.RustLayerException import cash.z.wallet.sdk.exception.WalletException +import cash.z.wallet.sdk.ext.SAPLING_ACTIVATION_HEIGHT import cash.z.wallet.sdk.ext.masked import cash.z.wallet.sdk.jni.RustBackendWelding import cash.z.wallet.sdk.secure.Wallet.WalletBirthday @@ -76,17 +76,34 @@ class Wallet( fun initialize( firstRunStartHeight: Int = SAPLING_ACTIVATION_HEIGHT ): Int { - twig("Initializing wallet for first run") - rustBackend.initDataDb(dataDbPath) - twig("seeding the database with sapling tree at height ${birthday.height}") - rustBackend.initBlocksTable(dataDbPath, birthday.height, birthday.hash, birthday.time, birthday.tree) + // TODO: find a better way to map these exceptions from the Rust side. For now, match error text :( - // store the spendingkey by leveraging the utilities provided during construction - val seed by seedProvider - val accountSpendingKeys = rustBackend.initAccountsTable(dataDbPath, seed, 1) - spendingKeyStore = accountSpendingKeys[0] + try { + rustBackend.initDataDb(dataDbPath) + twig("Initialized wallet for first run into file $dataDbPath") + } catch (e: Throwable) { + throw WalletException.FalseStart(e) + } - return Math.max(firstRunStartHeight, birthday.height) + try { + rustBackend.initBlocksTable(dataDbPath, birthday.height, birthday.hash, birthday.time, birthday.tree) + twig("seeded the database with sapling tree at height ${birthday.height} into file $dataDbPath") + } catch (t: Throwable) { + if (t.message?.contains("is not empty") == true) throw WalletException.AlreadyInitializedException(t) + else throw WalletException.FalseStart(t) + } + + try { + // store the spendingkey by leveraging the utilities provided during construction + val seed by seedProvider + val accountSpendingKeys = rustBackend.initAccountsTable(dataDbPath, seed, 1) + spendingKeyStore = accountSpendingKeys[0] + + twig("Initialized the accounts table into file $dataDbPath") + return Math.max(firstRunStartHeight, birthday.height) + } catch (e: Throwable) { + throw WalletException.FalseStart(e) + } } /** @@ -147,7 +164,7 @@ class Wallet( withContext(IO) { var result = -1L twigTask("creating raw transaction to send $value zatoshi to ${toAddress.masked()}") { - result = runCatching { + result = try { ensureParams(paramDestinationDir) twig("params exist at $paramDestinationDir! attempting to send...") rustBackend.sendToAddress( @@ -161,7 +178,10 @@ class Wallet( spendParams = SPEND_PARAM_FILE_NAME.toPath(), outputParams = OUTPUT_PARAM_FILE_NAME.toPath() ) - }.getOrDefault(result) + } catch (t: Throwable) { + twig("${t.message}") + throw t + } } twig("result of sendToAddress: $result") result From 305390b439c8c57d2ddf079dfb5413945c212168 Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:28:07 -0400 Subject: [PATCH 06/18] Update tests. --- .../cash/z/wallet/sdk/db/IntegrationTest.kt | 30 ++++--- .../z/wallet/sdk/util/AddressGeneratorUtil.kt | 79 +++++++++++++++++++ src/androidTest/resources/utils/seeds.txt | 3 + .../z/wallet/sdk/ext/CurrencyFormatter.kt | 2 +- .../java/cash/z/wallet/sdk/ext/ZcashSdk.kt | 59 ++++++++++++++ .../z/wallet/sdk/data/MockSynchronizerTest.kt | 4 +- 6 files changed, 165 insertions(+), 12 deletions(-) create mode 100644 src/androidTest/java/cash/z/wallet/sdk/util/AddressGeneratorUtil.kt create mode 100644 src/androidTest/resources/utils/seeds.txt create mode 100644 src/main/java/cash/z/wallet/sdk/ext/ZcashSdk.kt diff --git a/src/androidTest/java/cash/z/wallet/sdk/db/IntegrationTest.kt b/src/androidTest/java/cash/z/wallet/sdk/db/IntegrationTest.kt index b7d92615..607ccc4a 100644 --- a/src/androidTest/java/cash/z/wallet/sdk/db/IntegrationTest.kt +++ b/src/androidTest/java/cash/z/wallet/sdk/db/IntegrationTest.kt @@ -1,10 +1,14 @@ package cash.z.wallet.sdk.db -import android.text.format.DateUtils import androidx.test.platform.app.InstrumentationRegistry +import cash.z.wallet.sdk.block.CompactBlockDbStore +import cash.z.wallet.sdk.block.CompactBlockDownloader +import cash.z.wallet.sdk.block.CompactBlockProcessor +import cash.z.wallet.sdk.block.ProcessorConfig import cash.z.wallet.sdk.data.* import cash.z.wallet.sdk.jni.RustBackend import cash.z.wallet.sdk.secure.Wallet +import cash.z.wallet.sdk.service.LightWalletGrpcService import kotlinx.coroutines.runBlocking import org.junit.AfterClass import org.junit.Before @@ -21,13 +25,14 @@ class IntegrationTest { private val cacheDdName = "IntegrationCache41.db" private val context = InstrumentationRegistry.getInstrumentation().context - private lateinit var downloader: CompactBlockStream + private lateinit var downloader: CompactBlockDownloader private lateinit var processor: CompactBlockProcessor private lateinit var wallet: Wallet @Before fun setup() { deleteDbs() + Twig.plant(TroubleshootingTwig()) } private fun deleteDbs() { @@ -38,14 +43,22 @@ class IntegrationTest { } } - @Test(timeout = 1L * DateUtils.MINUTE_IN_MILLIS/10) + @Test(timeout = 120_000L) fun testSync() = runBlocking { val rustBackend = RustBackend() rustBackend.initLogs() - val logger = TroubleshootingTwig() + val config = ProcessorConfig( + cacheDbPath = context.getDatabasePath(cacheDdName).absolutePath, + dataDbPath = context.getDatabasePath(dataDbName).absolutePath, + downloadBatchSize = 2000, + blockPollFrequencyMillis = 10_000L + ) - downloader = CompactBlockStream("10.0.2.2", 9067, logger) - processor = CompactBlockProcessor(context, rustBackend, cacheDdName, dataDbName, logger = logger) + val lightwalletService = LightWalletGrpcService("192.168.1.134") + val compactBlockStore = CompactBlockDbStore(context, config.cacheDbPath) + + downloader = CompactBlockDownloader(lightwalletService, compactBlockStore) + processor = CompactBlockProcessor(config, downloader, repository, rustBackend) repository = PollingTransactionRepository(context, dataDbName, 10_000L) wallet = Wallet( context, @@ -59,16 +72,15 @@ class IntegrationTest { // repository.start(this) synchronizer = SdkSynchronizer( - downloader, processor, repository, - ActiveTransactionManager(repository, downloader.connection, wallet, logger), + ActiveTransactionManager(repository, lightwalletService, wallet), wallet, 1000 ).start(this) for(i in synchronizer.progress()) { - logger.twig("made progress: $i") + twig("made progress: $i") } } diff --git a/src/androidTest/java/cash/z/wallet/sdk/util/AddressGeneratorUtil.kt b/src/androidTest/java/cash/z/wallet/sdk/util/AddressGeneratorUtil.kt new file mode 100644 index 00000000..ed0e6449 --- /dev/null +++ b/src/androidTest/java/cash/z/wallet/sdk/util/AddressGeneratorUtil.kt @@ -0,0 +1,79 @@ +package cash.z.wallet.sdk.db + +import androidx.test.platform.app.InstrumentationRegistry +import cash.z.wallet.sdk.data.SampleSeedProvider +import cash.z.wallet.sdk.data.TroubleshootingTwig +import cash.z.wallet.sdk.data.Twig +import cash.z.wallet.sdk.jni.RustBackend +import cash.z.wallet.sdk.secure.Wallet +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.runBlocking +import okio.Okio +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import java.io.IOException +import kotlin.properties.Delegates +import kotlin.properties.ReadWriteProperty + +@ExperimentalCoroutinesApi +class AddressGeneratorUtil { + + private val dataDbName = "AddressUtilData.db" + private val context = InstrumentationRegistry.getInstrumentation().context + private val rustBackend = RustBackend() + + private lateinit var wallet: Wallet + + @Before + fun setup() { + Twig.plant(TroubleshootingTwig()) + rustBackend.initLogs() + } + + private fun deleteDb() { + context.getDatabasePath(dataDbName).absoluteFile.delete() + } + + @Test + fun generateAddresses() = runBlocking { + readLines().collect { seed -> + val keyStore = initWallet(seed) + val address = wallet.getAddress() + val pk by keyStore + println("xrxrx2\t$seed\t$address\t$pk") + } + Thread.sleep(5000) + assertEquals("foo", "bar") + } + + @Throws(IOException::class) + fun readLines() = flow { + val seedFile = javaClass.getResourceAsStream("/utils/seeds.txt") + Okio.buffer(Okio.source(seedFile)).use { source -> + var line: String? = source.readUtf8Line() + while (line != null) { + emit(line) + line = source.readUtf8Line() + } + } + } + + private fun initWallet(seed: String): ReadWriteProperty { + deleteDb() + val spendingKeyProvider = Delegates.notNull() + wallet = Wallet( + context, + rustBackend, + context.getDatabasePath(dataDbName).absolutePath, + context.cacheDir.absolutePath, + arrayOf(0), + SampleSeedProvider(seed), + spendingKeyProvider + ) + wallet.initialize() + return spendingKeyProvider + } +} \ No newline at end of file diff --git a/src/androidTest/resources/utils/seeds.txt b/src/androidTest/resources/utils/seeds.txt new file mode 100644 index 00000000..2d1c9d9d --- /dev/null +++ b/src/androidTest/resources/utils/seeds.txt @@ -0,0 +1,3 @@ +seed-1 +seed-2 +seed-3 \ No newline at end of file diff --git a/src/main/java/cash/z/wallet/sdk/ext/CurrencyFormatter.kt b/src/main/java/cash/z/wallet/sdk/ext/CurrencyFormatter.kt index 7dbaf806..eb8d3e5e 100644 --- a/src/main/java/cash/z/wallet/sdk/ext/CurrencyFormatter.kt +++ b/src/main/java/cash/z/wallet/sdk/ext/CurrencyFormatter.kt @@ -11,7 +11,7 @@ import java.util.* //TODO: provide a dynamic way to configure this globally for the SDK // For now, just make these vars so at least they could be modified in one place object Conversions { - var ONE_ZEC_IN_ZATOSHI = BigDecimal(100_000_000.0, MathContext.DECIMAL128) + var ONE_ZEC_IN_ZATOSHI = BigDecimal(ZATOSHI, MathContext.DECIMAL128) var ZEC_FORMATTER = NumberFormat.getInstance(Locale.getDefault()).apply { roundingMode = RoundingMode.HALF_EVEN maximumFractionDigits = 6 diff --git a/src/main/java/cash/z/wallet/sdk/ext/ZcashSdk.kt b/src/main/java/cash/z/wallet/sdk/ext/ZcashSdk.kt new file mode 100644 index 00000000..4614b6bc --- /dev/null +++ b/src/main/java/cash/z/wallet/sdk/ext/ZcashSdk.kt @@ -0,0 +1,59 @@ +package cash.z.wallet.sdk.ext + +// +// Constants +// + +/** + * Miner's fee in zatoshi. + */ +const val MINERS_FEE_ZATOSHI = 10_000L + +/** + * The number of zatoshi that equal 1 ZEC. + */ +const val ZATOSHI = 100_000_000L + +/** + * The height of the first sapling block. When it comes to shielded transactions, we do not need to consider any blocks + * prior to this height, at all. + */ +const val SAPLING_ACTIVATION_HEIGHT = 280_000 + +/** + * The theoretical maximum number of blocks in a reorg, due to other bottlenecks in the protocol design. + */ +const val MAX_REORG_SIZE = 100 + + +// +// Defaults +// + +/** + * Default size of batches of blocks to request from the compact block service. + */ +const val DEFAULT_BATCH_SIZE = 100 + +/** + * Default amount of time, in milliseconds, to poll for new blocks. Typically, this should be about half the average + * block time. + */ +const val DEFAULT_POLL_INTERVAL = 75_000L + +/** + * Default attempts at retrying. + */ +const val DEFAULT_RETRIES = 5 + +/** + * Default number of blocks to rewind when a chain reorg is detected. This should be large enough to recover from the + * reorg but smaller than the theoretical max reorg size of 100. + */ +const val DEFAULT_REWIND_DISTANCE = 10 + +/** + * The number of blocks to allow before considering our data to be stale. This usually helps with what to do when + * returning from the background and is exposed via the Synchronizer's isStale function. + */ +const val DEFAULT_STALE_TOLERANCE = 10 \ No newline at end of file diff --git a/src/test/java/cash/z/wallet/sdk/data/MockSynchronizerTest.kt b/src/test/java/cash/z/wallet/sdk/data/MockSynchronizerTest.kt index 2eb0f8da..05c222f5 100644 --- a/src/test/java/cash/z/wallet/sdk/data/MockSynchronizerTest.kt +++ b/src/test/java/cash/z/wallet/sdk/data/MockSynchronizerTest.kt @@ -141,7 +141,7 @@ internal class MockSynchronizerTest { @Test fun `balance matches transactions without sends`() = runBlocking { - val balances = fastSynchronizer.start(fastSynchronizer).balance() + val balances = fastSynchronizer.start(fastSynchronizer).balances() var transactions = listOf() while (transactions.count() < 10) { transactions = fastSynchronizer.allTransactions().receive() @@ -153,7 +153,7 @@ internal class MockSynchronizerTest { @Test fun `balance matches transactions with sends`() = runBlocking { var transactions = listOf() - val balances = fastSynchronizer.start(fastSynchronizer).balance() + val balances = fastSynchronizer.start(fastSynchronizer).balances() val transactionChannel = fastSynchronizer.allTransactions() while (transactions.count() < 10) { fastSynchronizer.sendToAddress(Random.nextLong(1L..10_000_000_000), validAddress) From 7daeb207557fe6200d3f853091dc301b13e2c5c7 Mon Sep 17 00:00:00 2001 From: Kevin Gorham Date: Fri, 14 Jun 2019 19:29:54 -0400 Subject: [PATCH 07/18] Sample App: Add sample app for deriving addresses and spending keys. --- samples/addressAndKeys/.gitignore | 14 ++ samples/addressAndKeys/app/.gitignore | 1 + samples/addressAndKeys/app/build.gradle | 37 ++++ samples/addressAndKeys/app/proguard-rules.pro | 21 +++ .../app/src/main/AndroidManifest.xml | 22 +++ .../cash/z/wallet/sdk/sample/address/App.kt | 14 ++ .../z/wallet/sdk/sample/address/Injection.kt | 35 ++++ .../wallet/sdk/sample/address/MainActivity.kt | 96 ++++++++++ .../sdk/sample/address/ScopedActivity.kt | 26 +++ .../drawable-v24/ic_launcher_foreground.xml | 34 ++++ .../res/drawable/ic_launcher_background.xml | 74 ++++++++ .../app/src/main/res/layout/activity_main.xml | 29 +++ .../res/mipmap-anydpi-v26/ic_launcher.xml | 5 + .../mipmap-anydpi-v26/ic_launcher_round.xml | 5 + .../src/main/res/mipmap-hdpi/ic_launcher.png | Bin 0 -> 2963 bytes .../res/mipmap-hdpi/ic_launcher_round.png | Bin 0 -> 4905 bytes .../src/main/res/mipmap-mdpi/ic_launcher.png | Bin 0 -> 2060 bytes .../res/mipmap-mdpi/ic_launcher_round.png | Bin 0 -> 2783 bytes .../src/main/res/mipmap-xhdpi/ic_launcher.png | Bin 0 -> 4490 bytes .../res/mipmap-xhdpi/ic_launcher_round.png | Bin 0 -> 6895 bytes .../main/res/mipmap-xxhdpi/ic_launcher.png | Bin 0 -> 6387 bytes .../res/mipmap-xxhdpi/ic_launcher_round.png | Bin 0 -> 10413 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.png | Bin 0 -> 9128 bytes .../res/mipmap-xxxhdpi/ic_launcher_round.png | Bin 0 -> 15132 bytes .../app/src/main/res/values/colors.xml | 6 + .../app/src/main/res/values/strings.xml | 3 + .../app/src/main/res/values/styles.xml | 11 ++ samples/addressAndKeys/build.gradle | 26 +++ samples/addressAndKeys/gradle.properties | 21 +++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 54329 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + samples/addressAndKeys/gradlew | 172 ++++++++++++++++++ samples/addressAndKeys/gradlew.bat | 84 +++++++++ samples/addressAndKeys/settings.gradle | 3 + 34 files changed, 745 insertions(+) create mode 100644 samples/addressAndKeys/.gitignore create mode 100644 samples/addressAndKeys/app/.gitignore create mode 100644 samples/addressAndKeys/app/build.gradle create mode 100644 samples/addressAndKeys/app/proguard-rules.pro create mode 100644 samples/addressAndKeys/app/src/main/AndroidManifest.xml create mode 100644 samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/App.kt create mode 100644 samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/Injection.kt create mode 100644 samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/MainActivity.kt create mode 100644 samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/ScopedActivity.kt create mode 100644 samples/addressAndKeys/app/src/main/res/drawable-v24/ic_launcher_foreground.xml create mode 100644 samples/addressAndKeys/app/src/main/res/drawable/ic_launcher_background.xml create mode 100644 samples/addressAndKeys/app/src/main/res/layout/activity_main.xml create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-hdpi/ic_launcher.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-hdpi/ic_launcher_round.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-mdpi/ic_launcher.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-mdpi/ic_launcher_round.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-xhdpi/ic_launcher.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-xxhdpi/ic_launcher.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png create mode 100644 samples/addressAndKeys/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png create mode 100644 samples/addressAndKeys/app/src/main/res/values/colors.xml create mode 100644 samples/addressAndKeys/app/src/main/res/values/strings.xml create mode 100644 samples/addressAndKeys/app/src/main/res/values/styles.xml create mode 100644 samples/addressAndKeys/build.gradle create mode 100644 samples/addressAndKeys/gradle.properties create mode 100644 samples/addressAndKeys/gradle/wrapper/gradle-wrapper.jar create mode 100644 samples/addressAndKeys/gradle/wrapper/gradle-wrapper.properties create mode 100755 samples/addressAndKeys/gradlew create mode 100644 samples/addressAndKeys/gradlew.bat create mode 100644 samples/addressAndKeys/settings.gradle diff --git a/samples/addressAndKeys/.gitignore b/samples/addressAndKeys/.gitignore new file mode 100644 index 00000000..603b1407 --- /dev/null +++ b/samples/addressAndKeys/.gitignore @@ -0,0 +1,14 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx diff --git a/samples/addressAndKeys/app/.gitignore b/samples/addressAndKeys/app/.gitignore new file mode 100644 index 00000000..796b96d1 --- /dev/null +++ b/samples/addressAndKeys/app/.gitignore @@ -0,0 +1 @@ +/build diff --git a/samples/addressAndKeys/app/build.gradle b/samples/addressAndKeys/app/build.gradle new file mode 100644 index 00000000..9ed8781b --- /dev/null +++ b/samples/addressAndKeys/app/build.gradle @@ -0,0 +1,37 @@ +apply plugin: 'com.android.application' +apply plugin: 'kotlin-android' +apply plugin: 'kotlin-android-extensions' + +android { + compileSdkVersion 28 + defaultConfig { + applicationId "cash.z.wallet.sdk.sample.address" + minSdkVersion 21 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + missingDimensionStrategy "network", "zcashtestnet" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + matchingFallbacks = ['zcashtestnetRelease'] + } + } +} + +dependencies { + compile project(path: ':sdk') +// api project(path: ':sdk', configuration: 'default') + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${versions.kotlin}" + implementation 'androidx.appcompat:appcompat:1.0.2' + implementation 'androidx.core:core-ktx:1.0.2' + implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:${versions.coroutines}" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:${versions.coroutines}" + testImplementation 'junit:junit:4.12' + androidTestImplementation 'androidx.test.ext:junit:1.1.1' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' +} diff --git a/samples/addressAndKeys/app/proguard-rules.pro b/samples/addressAndKeys/app/proguard-rules.pro new file mode 100644 index 00000000..f1b42451 --- /dev/null +++ b/samples/addressAndKeys/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/samples/addressAndKeys/app/src/main/AndroidManifest.xml b/samples/addressAndKeys/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..dd6f1e4d --- /dev/null +++ b/samples/addressAndKeys/app/src/main/AndroidManifest.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/App.kt b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/App.kt new file mode 100644 index 00000000..c0d02831 --- /dev/null +++ b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/App.kt @@ -0,0 +1,14 @@ +package cash.z.wallet.sdk.sample.address + +import android.app.Application + +class App : Application() { + override fun onCreate() { + instance = this + super.onCreate() + } + + companion object { + lateinit var instance: App + } +} \ No newline at end of file diff --git a/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/Injection.kt b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/Injection.kt new file mode 100644 index 00000000..12e49bf5 --- /dev/null +++ b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/Injection.kt @@ -0,0 +1,35 @@ +package cash.z.wallet.sdk.sample.address + +import cash.z.wallet.sdk.jni.RustBackend +import cash.z.wallet.sdk.jni.RustBackendWelding +import cash.z.wallet.sdk.secure.Wallet +import kotlin.properties.ReadOnlyProperty +import kotlin.properties.ReadWriteProperty + +object Injection { + private val rustBackend: RustBackendWelding = RustBackend() + private const val dataDbName = "AddressSampleData.db" + + fun provideWallet( + seedProvider: ReadOnlyProperty, + spendingKeyProvider: ReadWriteProperty + ): Wallet { + // simulate new session for each call + App.instance.getDatabasePath(dataDbName).absoluteFile.delete() + + return Wallet( + App.instance, + provideRustBackend(), + App.instance.getDatabasePath(dataDbName).absolutePath, + App.instance.cacheDir.absolutePath, + arrayOf(0), + seedProvider, + spendingKeyProvider + ) + } + + fun provideRustBackend(): RustBackendWelding { + return rustBackend + } +} + diff --git a/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/MainActivity.kt b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/MainActivity.kt new file mode 100644 index 00000000..b47f118d --- /dev/null +++ b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/MainActivity.kt @@ -0,0 +1,96 @@ +package cash.z.wallet.sdk.sample.address + +import android.os.Bundle +import android.view.View +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity +import cash.z.wallet.sdk.data.SampleSeedProvider +import cash.z.wallet.sdk.data.TroubleshootingTwig +import cash.z.wallet.sdk.data.Twig +import cash.z.wallet.sdk.jni.RustBackend +import cash.z.wallet.sdk.jni.RustBackendWelding +import cash.z.wallet.sdk.secure.Wallet +import kotlinx.coroutines.runBlocking +import kotlin.properties.Delegates + +/** + * Sample app that shows how to access the address and spending key. + */ +class MainActivity : AppCompatActivity() { + private val seedFromSecureElement = "testreferencebob" + private lateinit var wallet: Wallet + private lateinit var rustBackend: RustBackendWelding + private lateinit var addressInfo: TextView + + // Secure storage is out of scope for this example (wallet makers know how to securely store things) + // However, any class can implement the required interface for these dependencies. The expectation is that a wallet + // maker would wrap an existing class with something that implements the property interface to access data. These + // dependencies would then point to those wrappers. + private val mockSecureStorage = Delegates.notNull() + private val mockSecureSeedProvider = SampleSeedProvider(seedFromSecureElement) + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + Twig.plant(TroubleshootingTwig()) + + addressInfo = findViewById(R.id.text_address_info) + + rustBackend = Injection.provideRustBackend() + wallet = Injection.provideWallet(mockSecureSeedProvider, mockSecureStorage) + wallet.initialize() + } + + override fun onResume() { + super.onResume() + + val address = wallet.getAddress() + + // The wallet does not provide the spending key. Any request for the key must be delegated to the secure storage + val key by mockSecureStorage + + val info = """ + seed: + $seedFromSecureElement + -------------------------------------- + address: + $address + -------------------------------------- + spendingKey: + $key + """.trimIndent() + addressInfo.text = info + } + + fun onTestThings(view: View) { + testWalletSend() + } + + private fun testWalletSend() = runBlocking { + try { + val result = wallet.createRawSendTransaction(20_000L, wallet.getAddress(), "") + addressInfo.text = "\"Succeeded\" with value: $result" + } catch (t: Throwable) { + addressInfo.text = "Exception: ${t::class}\n\nMessage: ${t.message}" + } + } + + fun testRustSend() { + try { + val key by mockSecureStorage + rustBackend.sendToAddress( + dbData = "/data/user/0/cash.z.wallet.sdk.sample.address/cache/test_data_bob.db", // String, + account = 0, // Int, + extsk = key, // String, + to = wallet.getAddress(), // String, + value = 20_000L, // Long, + memo = "", // String, + spendParams = "/data/user/0/cash.z.wallet.sdk.sample.address/cache/sapling-spend.params", // String, + outputParams = "/data/user/0/cash.z.wallet.sdk.sample.address/cache/sapling-output.params" // String + ) + } catch (t: Throwable) { + addressInfo.text = "Exception: ${t::class}\n\nMessage: ${t.message}" + } + } + +} diff --git a/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/ScopedActivity.kt b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/ScopedActivity.kt new file mode 100644 index 00000000..9d578d4c --- /dev/null +++ b/samples/addressAndKeys/app/src/main/java/cash/z/wallet/sdk/sample/address/ScopedActivity.kt @@ -0,0 +1,26 @@ +package cash.z.wallet.sdk.sample.address + +import android.os.Bundle +import androidx.appcompat.app.AppCompatActivity +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlin.coroutines.CoroutineContext + +open class ScopedActivity : AppCompatActivity(), CoroutineScope { + private lateinit var job: Job + + override val coroutineContext: CoroutineContext + get() = job + Dispatchers.Main + + override fun onCreate(savedInstanceState: Bundle?) { + job = Job() + super.onCreate(savedInstanceState) + } + + override fun onDestroy() { + super.onDestroy() + job.cancel() + } + +} diff --git a/samples/addressAndKeys/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/samples/addressAndKeys/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 00000000..6348baae --- /dev/null +++ b/samples/addressAndKeys/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/samples/addressAndKeys/app/src/main/res/drawable/ic_launcher_background.xml b/samples/addressAndKeys/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 00000000..a0ad202f --- /dev/null +++ b/samples/addressAndKeys/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/addressAndKeys/app/src/main/res/layout/activity_main.xml b/samples/addressAndKeys/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000..345ce1a6 --- /dev/null +++ b/samples/addressAndKeys/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,29 @@ + + + + + +