Add support for inbound memos and full wallet restore.

src/main/java/cash/z/wallet/sdk/SdkSynchronizer.kt

@@ -208,8 +208,11 @@ class SdkSynchronizer internal constructor(
                 is Downloading, Initialized -> DOWNLOADING
                 is Validating -> VALIDATING
                 is Scanning -> SCANNING
+                is Enhancing -> ENHANCING
             }.let { synchronizerStatus ->
-                _status.send(synchronizerStatus)
+                //  ignore enhancing status for now
+                // TODO: clean this up and handle enhancing gracefully
+                if (synchronizerStatus != ENHANCING) _status.send(synchronizerStatus)
             }
         }.launchIn(this)
         processor.start()

src/main/java/cash/z/wallet/sdk/Synchronizer.kt

@@ -4,6 +4,7 @@ import androidx.paging.PagedList
 import cash.z.wallet.sdk.block.CompactBlockProcessor
 import cash.z.wallet.sdk.block.CompactBlockProcessor.WalletBalance
 import cash.z.wallet.sdk.entity.*
+import cash.z.wallet.sdk.rpc.Service
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.Flow
 
@@ -248,6 +249,12 @@ interface Synchronizer {
          */
         SCANNING,
 
+        /**
+         * Indicates that this Synchronizer is actively enhancing newly scanned blocks with
+         * additional transaction details, fetched from the server.
+         */
+        ENHANCING,
+
         /**
          * Indicates that this Synchronizer is fully up to date and ready for all wallet functions.
          * When set, a UI element may want to turn green. In this state, the balance can be trusted.

src/main/java/cash/z/wallet/sdk/block/CompactBlockDownloader.kt

@@ -69,5 +69,12 @@ open class CompactBlockDownloader(
         compactBlockStore.close()
     }
 
+    /**
+     * Fetch the details of a known transaction.
+     *
+     * @return the full transaction info.
+     */
+    fun fetchTransaction(txId: ByteArray) = lightwalletService.fetchTransaction(txId)
+
 }
 

src/main/java/cash/z/wallet/sdk/block/CompactBlockProcessor.kt

@@ -3,6 +3,7 @@ package cash.z.wallet.sdk.block
 import androidx.annotation.VisibleForTesting
 import cash.z.wallet.sdk.annotation.OpenForTesting
 import cash.z.wallet.sdk.block.CompactBlockProcessor.State.*
+import cash.z.wallet.sdk.entity.ConfirmedTransaction
 import cash.z.wallet.sdk.exception.CompactBlockProcessorException
 import cash.z.wallet.sdk.exception.RustLayerException
 import cash.z.wallet.sdk.ext.*
@@ -17,10 +18,13 @@ import cash.z.wallet.sdk.ext.ZcashSdk.SCAN_BATCH_SIZE
 import cash.z.wallet.sdk.jni.RustBackend
 import cash.z.wallet.sdk.jni.RustBackendWelding
 import cash.z.wallet.sdk.transaction.TransactionRepository
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Dispatchers.IO
 import kotlinx.coroutines.channels.ConflatedBroadcastChannel
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.asFlow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.withContext
 import java.util.concurrent.atomic.AtomicInteger
@@ -159,7 +163,9 @@ class CompactBlockProcessor(
             -1
         } else {
             downloadNewBlocks(currentInfo.lastDownloadRange)
-            validateAndScanNewBlocks(currentInfo.lastScanRange)
+            validateAndScanNewBlocks(currentInfo.lastScanRange).also {
+                enhanceTransactionDetails(currentInfo.lastScanRange)
+            }
         }
     }
 
@@ -168,6 +174,9 @@ class CompactBlockProcessor(
      * the scan/download ranges that require processing.
      */
     private suspend fun updateRanges() = withContext(IO)  {
+        // TODO: rethink this and make it easier to understand what's happening. Can we reduce this
+        // so that we only work with actual changing info rather than periodic snapshots? Do we need
+        // to calculate these derived values every time?
         ProcessorInfo(
             networkBlockHeight = downloader.getLatestBlockHeight(),
             lastScannedHeight = getLastScannedHeight(),
@@ -214,6 +223,37 @@ class CompactBlockProcessor(
         }
     }
 
+    private suspend fun enhanceTransactionDetails(lastScanRange: IntRange): Int {
+        Twig.sprout("enhancing")
+        twig("Enhancing transaction details for blocks $lastScanRange")
+        setState(Enhancing)
+        return try {
+            val newTxs = repository.findNewTransactions(lastScanRange)
+            if (newTxs == null) twig("no new transactions found in $lastScanRange")
+            newTxs?.onEach { newTransaction ->
+                if (newTransaction == null) twig("somehow, new transaction was null!!!")
+                else enhance(newTransaction)
+            }
+            twig("Done enhancing transaction details")
+            1
+        } catch (t: Throwable) {
+            twig("Failed to enhance due to $t")
+            t.printStackTrace()
+            -1
+        } finally {
+            Twig.clip("enhancing")
+        }
+    }
+
+    private suspend fun enhance(transaction: ConfirmedTransaction) = withContext(Dispatchers.IO) {
+        twig("START: enhancing transaction (id:${transaction.id}  block:${transaction.minedHeight})")
+        downloader.fetchTransaction(transaction.rawTransactionId)?.let { tx ->
+            twig("decrypting and storing transaction (id:${transaction.id}  block:${transaction.minedHeight})")
+            rustBackend.decryptAndStoreTransaction(tx.data.toByteArray())
+        } ?: twig("no transaction found. Nothing to enhance. This probably shouldn't happen.")
+        twig("DONE: enhancing transaction (id:${transaction.id}  block:${transaction.minedHeight})")
+    }
+
     /**
      * Confirm that the wallet data is properly setup for use.
      */
@@ -467,6 +507,15 @@ class CompactBlockProcessor(
          */
         class Scanned(val scannedRange:IntRange) : Connected, Syncing, State()
 
+        /**
+         * [State] for when transaction details are being retrieved. This typically means the wallet
+         * has downloaded and scanned blocks and is now processing any transactions that were
+         * discovered. Once a transaction is discovered, followup network requests are needed in
+         * order to retrieve memos or outbound transaction information, like the recipient address.
+         * The existing information we have about transactions is enhanced by the new information.
+         */
+        object Enhancing : Connected, Syncing, State()
+
         /**
          * [State] for when we have no connection to lightwalletd.
          */

src/main/java/cash/z/wallet/sdk/db/DerivedDataDb.kt

@@ -192,5 +192,48 @@ interface TransactionDao {
          LIMIT  :limit
     """)
     fun getAllTransactions(limit: Int = Int.MAX_VALUE): DataSource.Factory<Int, ConfirmedTransaction>
+
+    /**
+     * Query the transactions table over the given block range, this includes transactions that
+     * should not show up in most UIs. The intended purpose of this request is to find new
+     * transactions that need to be enhanced via follow-up requests to the server.
+     */
+    @Query("""
+        SELECT transactions.id_tx         AS id, 
+               transactions.block         AS minedHeight, 
+               transactions.tx_index      AS transactionIndex, 
+               transactions.txid          AS rawTransactionId, 
+               transactions.expiry_height AS expiryHeight, 
+               transactions.raw           AS raw, 
+               sent_notes.address         AS toAddress, 
+               CASE 
+                 WHEN transactions.raw IS NOT NULL THEN sent_notes.value 
+                 ELSE received_notes.value 
+               end                        AS value, 
+               CASE 
+                 WHEN transactions.raw IS NOT NULL THEN sent_notes.memo 
+                 ELSE received_notes.memo 
+               end                        AS memo, 
+               CASE 
+                 WHEN transactions.raw IS NOT NULL THEN sent_notes.id_note 
+                 ELSE received_notes.id_note 
+               end                        AS noteId, 
+               blocks.time                AS blockTimeInSeconds 
+        FROM   transactions 
+               LEFT JOIN received_notes 
+                      ON transactions.id_tx = received_notes.tx 
+               LEFT JOIN sent_notes 
+                      ON transactions.id_tx = sent_notes.tx 
+               LEFT JOIN blocks 
+                      ON transactions.block = blocks.height 
+        WHERE  :blockRangeStart <= minedheight AND minedheight <= :blockRangeEnd 
+        ORDER  BY ( minedheight IS NOT NULL ), 
+                  minedheight ASC, 
+                  blocktimeinseconds DESC, 
+                  id DESC 
+        LIMIT  :limit 
+    """)
+    suspend fun findAllTransactionsByRange(blockRangeStart: Int, blockRangeEnd: Int = blockRangeStart, limit: Int = Int.MAX_VALUE): List<ConfirmedTransaction>
+
 }
 

src/main/java/cash/z/wallet/sdk/service/LightWalletGrpcService.kt

@@ -68,6 +68,12 @@ class LightWalletGrpcService private constructor(
         channel.shutdownNow()
     }
 
+    override fun fetchTransaction(txId: ByteArray): Service.RawTransaction? {
+        channel.resetConnectBackoff()
+        return channel.createStub().getTransaction(Service.TxFilter.newBuilder().setHash(ByteString.copyFrom(txId)).build())
+    }
+
+
     //
     // Utilities
     //

src/main/java/cash/z/wallet/sdk/service/LightWalletService.kt

@@ -37,4 +37,11 @@ interface LightWalletService {
      * Cleanup any connections when the service is shutting down and not going to be used again.
      */
     fun shutdown()
+
+    /**
+     * Fetch the details of a known transaction.
+     *
+     * @return the full transaction info.
+     */
+    fun fetchTransaction(txId: ByteArray): Service.RawTransaction?
 }

src/main/java/cash/z/wallet/sdk/transaction/PagedTransactionRepository.kt

@@ -9,6 +9,7 @@ import androidx.sqlite.db.SupportSQLiteDatabase
 import cash.z.wallet.sdk.db.BlockDao
 import cash.z.wallet.sdk.db.DerivedDataDb
 import cash.z.wallet.sdk.db.TransactionDao
+import cash.z.wallet.sdk.entity.ConfirmedTransaction
 import cash.z.wallet.sdk.ext.ZcashSdk
 import cash.z.wallet.sdk.ext.android.toFlowPagedList
 import cash.z.wallet.sdk.ext.android.toRefreshable
@@ -76,6 +77,10 @@ open class PagedTransactionRepository(
         transactions.findEncodedTransactionById(txId)
     }
 
+    override suspend fun findNewTransactions(blockHeightRange: IntRange): List<ConfirmedTransaction> =
+        transactions.findAllTransactionsByRange(blockHeightRange.first, blockHeightRange.last)
+
+
     override suspend fun findMinedHeight(rawTransactionId: ByteArray) = withContext(IO) {
         transactions.findMinedHeight(rawTransactionId)
     }

src/main/java/cash/z/wallet/sdk/transaction/TransactionRepository.kt

@@ -32,6 +32,19 @@ interface TransactionRepository {
      */
     suspend fun findEncodedTransactionById(txId: Long): EncodedTransaction?
 
+    /**
+     * Find all the newly scanned transactions in the given range, including transactions (like
+     * change or those only identified by nullifiers) which should not appear in the UI. This method
+     * is intended for use after a scan, in order to collect all the transactions that were
+     * discovered and then enhance them with additional details. It returns a list to signal that
+     * the intention is not to add them to a recyclerview or otherwise show in the UI.
+     *
+     * @param blockHeightRange the range of blocks to check for transactions.
+     *
+     * @return a list of transactions that were mined in the given range, inclusive.
+     */
+    suspend fun findNewTransactions(blockHeightRange: IntRange): List<ConfirmedTransaction>
+
     /**
      * Find the mined height that matches the given raw tx_id in bytes. This is useful for matching
      * a pending transaction with one that we've decrypted from the blockchain.