[#1475] Adopt transaction data requests

- RustBackend extended for the new 2 rust methods 'transactionDataRequests' and 'setTransactionStatus' - lightwalletservice extended to add a new method 'getTaddressTxids' - Enhance action has been refactored to handle transactionDataRequests [#1475] Adopt transaction data requests - fixes [#1475] Adopt transaction data requests - Error codes for specific rust and service errors defined - Fix for the txId [#1475] Adopt transaction data requests - Checkpoints added - Code cleanup [#1475] Adopt transaction data requests - bugfixes in the ffi [#1475] Adopt transaction data requests - FFI with fixes [#1475] Adopt transaction data requests - Another FFI update with fixes, this time the final [#1475] Adopt transaction data requests - Fix for the not recognized state of the transaction for FetchTransaction(txId:) [#1475] Adopt transaction data requests - Code cleaned up and polished [#1475] Adopt transaction data requests - Changelog updated
Electric-Coin-Company · Aug 18, 2024 · 4dc0b2e · 4dc0b2e
1 parent a3e15f0
commit 4dc0b2e
Show file tree

Hide file tree

Showing 36 changed files with 451 additions and 41 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,21 @@ and this library adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `Synchronizer.exchangeRateUSDStream`. Prices are queried over Tor (to hide the wallet's
   IP address).
 
+## Changed
+
+### [#1475] Adopt transaction data requests
+- The transaction history is now processed using `transaction data requests`, which are fetched every 1,000 blocks during longer syncs or with each sync loop when a new block is mined.
+
+## Checkpoints
+
+Mainnet
+
+````
+Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2562500.json
+...
+Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2610000.json
+````
+
 # 2.1.12 - 2024-07-04
 
 ## Fixed

diff --git a/...cashLightClientSample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/...cashLightClientSample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -176,7 +176,7 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/zcash-hackworks/zcash-light-client-ffi",
       "state" : {
-        "revision" : "4de1b42f99aebfc5e4f0340da8a66a4f719db9a6"
+        "revision" : "3ccafcddfe51918239d157fd839476959413840f"
       }
     }
   ],

diff --git a/Example/ZcashLightClientSample/ZcashLightClientSample/DemoAppConfig.swift b/Example/ZcashLightClientSample/ZcashLightClientSample/DemoAppConfig.swift
@@ -21,14 +21,14 @@ enum DemoAppConfig {
     static let host = ZcashSDK.isMainnet ? "zec.rocks" : "lightwalletd.testnet.electriccoin.co"
     static let port: Int = 443
 
-    static let defaultBirthdayHeight: BlockHeight = ZcashSDK.isMainnet ? 935000 : 1386000
+    static let defaultBirthdayHeight: BlockHeight = ZcashSDK.isMainnet ? 2610000 : 2610000
 
 //    static let defaultSeed = try! Mnemonic.deterministicSeedBytes(from: """
 //        wish puppy smile loan doll curve hole maze file ginger hair nose key relax knife witness cannon grab despair throw review deal slush frame
 //        """)
 
     static let defaultSeed = try! Mnemonic.deterministicSeedBytes(from: """
-        live combine flight accident slow soda mind bright absent bid hen shy decade biology amazing mix enlist ensure biology rhythm snap duty soap armor
+        rhythm tide ignore unknown penalty helmet square secret nature sea ethics hint only since behind spawn crater ozone festival birth dynamic idea ceiling arch
         """)
 
     static let otherSynchronizers: [SynchronizerInitData] = [

diff --git a/Package.swift b/Package.swift
@@ -18,7 +18,8 @@ let package = Package(
         .package(url: "https://github.com/stephencelis/SQLite.swift.git", from: "0.14.1"),
         // .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", exact: "0.8.1")
         // Compiled from 2516a94f8bdc540d951c38b66e9c07e2b8c29cb4
-        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "4de1b42f99aebfc5e4f0340da8a66a4f719db9a6")
+//        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", branch: "ffi_transaction_requests_preview")
+        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "3ccafcddfe51918239d157fd839476959413840f")
     ],
     targets: [
         .target(

diff --git a/Sources/ZcashLightClientKit/Block/Enhance/BlockEnhancer.swift b/Sources/ZcashLightClientKit/Block/Enhance/BlockEnhancer.swift
@@ -60,21 +60,24 @@ struct BlockEnhancerImpl {
     let rustBackend: ZcashRustBackendWelding
     let transactionRepository: TransactionRepository
     let metrics: SDKMetrics
+    let service: LightWalletService
     let logger: Logger
 
-    private func enhance(transaction: ZcashTransaction.Overview) async throws -> ZcashTransaction.Overview {
-        logger.debug("Zoom.... Enhance... Tx: \(transaction.rawID.toHexStringTxId())")
+    private func enhance(txId: Data) async throws -> ZcashTransaction.Overview {
+        logger.debug("Zoom.... Enhance... Tx: \(txId.toHexStringTxId())")
 
-        let fetchedTransaction = try await blockDownloaderService.fetchTransaction(txId: transaction.rawID)
+        let fetchedTransaction = try await blockDownloaderService.fetchTransaction(txId: txId)
 
-        let transactionID = fetchedTransaction.rawID.toHexStringTxId()
-        let block = String(describing: transaction.minedHeight)
-        logger.debug("Decrypting and storing transaction id: \(transactionID) block: \(block)")
-
-        try await rustBackend.decryptAndStoreTransaction(
-            txBytes: fetchedTransaction.raw.bytes,
-            minedHeight: Int32(fetchedTransaction.minedHeight)
-        )
+        if fetchedTransaction.minedHeight == -1 {
+            try await rustBackend.setTransactionStatus(txId: fetchedTransaction.rawID, status: .txidNotRecognized)
+        } else if fetchedTransaction.minedHeight == 0 {
+            try await rustBackend.setTransactionStatus(txId: fetchedTransaction.rawID, status: .notInMainChain)
+        } else if fetchedTransaction.minedHeight > 0 {
+            try await rustBackend.decryptAndStoreTransaction(
+                txBytes: fetchedTransaction.raw.bytes,
+                minedHeight: Int32(fetchedTransaction.minedHeight)
+            )
+        }
 
         return try await transactionRepository.find(rawID: fetchedTransaction.rawID)
     }
@@ -92,42 +95,56 @@ extension BlockEnhancerImpl: BlockEnhancer {
         // fetch transactions
         do {
             let startTime = Date()
-            let transactions = try await transactionRepository.find(in: range, limit: Int.max, kind: .all)
+            let transactionDataRequests = try await rustBackend.transactionDataRequests()
 
-            guard !transactions.isEmpty else {
-                logger.debug("no transactions detected on range: \(range.lowerBound)...\(range.upperBound)")
-                logger.sync("No transactions detected on range: \(range.lowerBound)...\(range.upperBound)")
+            guard !transactionDataRequests.isEmpty else {
+                logger.debug("No transaction data requests detected.")
+                logger.sync("No transaction data requests detected.")
                 return nil
             }
 
             let chainTipHeight = try await blockDownloaderService.latestBlockHeight()
-
             let newlyMinedLowerBound = chainTipHeight - ZcashSDK.expiryOffset
-
             let newlyMinedRange = newlyMinedLowerBound...chainTipHeight
 
-            for index in 0 ..< transactions.count {
-                let transaction = transactions[index]
+            for index in 0 ..< transactionDataRequests.count {
+                let transactionDataRequest = transactionDataRequests[index]
                 var retry = true
-                
+
                 while retry && retries < maxRetries {
                     try Task.checkCancellation()
                     do {
-                        let confirmedTx = try await enhance(transaction: transaction)
-                        retry = false
-
-                        let progress = EnhancementProgress(
-                            totalTransactions: transactions.count,
-                            enhancedTransactions: index + 1,
-                            lastFoundTransaction: confirmedTx,
-                            range: range,
-                            newlyMined: confirmedTx.isSentTransaction && newlyMinedRange.contains(confirmedTx.minedHeight ?? 0)
-                        )
-
-                        await didEnhance(progress)
+                        switch transactionDataRequest {
+                        case .getStatus(let txId), .enhancement(let txId):
+                            let confirmedTx = try await enhance(txId: txId.data)
+                            retry = false
+
+                            let progress = EnhancementProgress(
+                                totalTransactions: transactionDataRequests.count,
+                                enhancedTransactions: index + 1,
+                                lastFoundTransaction: confirmedTx,
+                                range: range,
+                                newlyMined: confirmedTx.isSentTransaction && newlyMinedRange.contains(confirmedTx.minedHeight ?? 0)
+                            )
+
+                            await didEnhance(progress)
+                        case .spendsFromAddress(let sfa):
+                            var filter = TransparentAddressBlockFilter()
+                            filter.address = sfa.address
+                            filter.range = BlockRange(startHeight: Int(sfa.blockRangeStart), endHeight: Int(sfa.blockRangeEnd))
+                            let stream = service.getTaddressTxids(filter)
+
+                            for try await rawTransaction in stream {
+                                try await rustBackend.decryptAndStoreTransaction(
+                                    txBytes: rawTransaction.data.bytes,
+                                    minedHeight: Int32(rawTransaction.height)
+                                )
+                            }
+                            retry = false
+                        }
                     } catch {
                         retries += 1
-                        logger.error("could not enhance txId \(transaction.rawID.toHexStringTxId()) - Error: \(error)")
+                        logger.error("could not enhance transactionDataRequest \(transactionDataRequest) - Error: \(error)")
                         if retries > maxRetries {
                             throw error
                         }
@@ -137,7 +154,7 @@ extension BlockEnhancerImpl: BlockEnhancer {
 
             let endTime = Date()
             let diff = endTime.timeIntervalSince1970 - startTime.timeIntervalSince1970
-            let logMsg = "Enhanced \(transactions.count) transaction(s) in \(diff) for range \(range.lowerBound)...\(range.upperBound)"
+            let logMsg = "Enhanced \(transactionDataRequests.count) transaction data requests in \(diff)"
             logger.sync(logMsg)
             metrics.actionDetail(logMsg, for: .enhance)
         } catch {

diff --git a/Sources/ZcashLightClientKit/Error/ZcashError.swift b/Sources/ZcashLightClientKit/Error/ZcashError.swift
@@ -61,6 +61,9 @@ public enum ZcashError: Equatable, Error {
     /// LightWalletService.getSubtreeRoots failed.
     /// ZSRVC0009
     case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getTaddressTxids failed.
+    /// ZSRVC0010
+    case serviceGetTaddressTxidsFailed(_ error: LightWalletServiceError)
     /// SimpleConnectionProvider init of Connection failed.
     /// ZSCPC0001
     case simpleConnectionProvider(_ error: Error)
@@ -352,6 +355,11 @@ public enum ZcashError: Equatable, Error {
     /// sourcery: code="ZRUST0063"
     /// ZRUST0063
     case rustTorClientGet(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0064"
+    /// ZRUST0064
+    case rustTransactionDataRequests(_ rustError: String)
     /// SQLite query failed when fetching all accounts from the database.
     /// - `sqliteError` is error produced by SQLite library.
     /// ZADAO0001
@@ -646,6 +654,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchUTXOsFailed: return "LightWalletService.fetchUTXOs failed."
         case .serviceBlockStreamFailed: return "LightWalletService.blockStream failed."
         case .serviceSubtreeRootsStreamFailed: return "LightWalletService.getSubtreeRoots failed."
+        case .serviceGetTaddressTxidsFailed: return "LightWalletService.getTaddressTxids failed."
         case .simpleConnectionProvider: return "SimpleConnectionProvider init of Connection failed."
         case .saplingParamsInvalidSpendParams: return "Downloaded file with sapling spending parameters isn't valid."
         case .saplingParamsInvalidOutputParams: return "Downloaded file with sapling output parameters isn't valid."
@@ -720,6 +729,7 @@ public enum ZcashError: Equatable, Error {
         case .rustPutOrchardSubtreeRoots: return "Error from rust layer when calling ZcashRustBackend.putOrchardSubtreeRoots"
         case .rustTorClientInit: return "Error from rust layer when calling TorClient.init"
         case .rustTorClientGet: return "Error from rust layer when calling TorClient.get"
+        case .rustTransactionDataRequests: return "Error from rust layer when calling ZcashRustBackend.transactionDataRequests"
         case .accountDAOGetAll: return "SQLite query failed when fetching all accounts from the database."
         case .accountDAOGetAllCantDecode: return "Fetched accounts from SQLite but can't decode them."
         case .accountDAOFindBy: return "SQLite query failed when seaching for accounts in the database."
@@ -829,6 +839,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchUTXOsFailed: return .serviceFetchUTXOsFailed
         case .serviceBlockStreamFailed: return .serviceBlockStreamFailed
         case .serviceSubtreeRootsStreamFailed: return .serviceSubtreeRootsStreamFailed
+        case .serviceGetTaddressTxidsFailed: return .serviceGetTaddressTxidsFailed
         case .simpleConnectionProvider: return .simpleConnectionProvider
         case .saplingParamsInvalidSpendParams: return .saplingParamsInvalidSpendParams
         case .saplingParamsInvalidOutputParams: return .saplingParamsInvalidOutputParams
@@ -903,6 +914,7 @@ public enum ZcashError: Equatable, Error {
         case .rustPutOrchardSubtreeRoots: return .rustPutOrchardSubtreeRoots
         case .rustTorClientInit: return .rustTorClientInit
         case .rustTorClientGet: return .rustTorClientGet
+        case .rustTransactionDataRequests: return .rustTransactionDataRequests
         case .accountDAOGetAll: return .accountDAOGetAll
         case .accountDAOGetAllCantDecode: return .accountDAOGetAllCantDecode
         case .accountDAOFindBy: return .accountDAOFindBy

diff --git a/Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift b/Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift
@@ -41,6 +41,8 @@ public enum ZcashErrorCode: String {
     case serviceBlockStreamFailed = "ZSRVC0000"
     /// LightWalletService.getSubtreeRoots failed.
     case serviceSubtreeRootsStreamFailed = "ZSRVC0009"
+    /// LightWalletService.getTaddressTxids failed.
+    case serviceGetTaddressTxidsFailed = "ZSRVC0010"
     /// SimpleConnectionProvider init of Connection failed.
     case simpleConnectionProvider = "ZSCPC0001"
     /// Downloaded file with sapling spending parameters isn't valid.
@@ -189,6 +191,8 @@ public enum ZcashErrorCode: String {
     case rustTorClientInit = "ZRUST0062"
     /// Error from rust layer when calling TorClient.get
     case rustTorClientGet = "ZRUST0063"
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    case rustTransactionDataRequests = "ZRUST0064"
     /// SQLite query failed when fetching all accounts from the database.
     case accountDAOGetAll = "ZADAO0001"
     /// Fetched accounts from SQLite but can't decode them.

diff --git a/Sources/ZcashLightClientKit/Error/ZcashErrorCodeDefinition.swift b/Sources/ZcashLightClientKit/Error/ZcashErrorCodeDefinition.swift
@@ -80,6 +80,9 @@ enum ZcashErrorDefinition {
     /// LightWalletService.getSubtreeRoots failed.
     // sourcery: code="ZSRVC0009"
     case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getTaddressTxids failed.
+    // sourcery: code="ZSRVC0010"
+    case serviceGetTaddressTxidsFailed(_ error: LightWalletServiceError)
 
     // MARK: SQLite connection
 
@@ -375,6 +378,10 @@ enum ZcashErrorDefinition {
     /// - `rustError` contains error generated by the rust layer.
     /// sourcery: code="ZRUST0063"
     case rustTorClientGet(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0064"
+    case rustTransactionDataRequests(_ rustError: String)
 
     // MARK: - Account DAO
 

diff --git a/Sources/ZcashLightClientKit/Model/Proposal.swift b/Sources/ZcashLightClientKit/Model/Proposal.swift
@@ -35,7 +35,7 @@ public extension Proposal {
     /// passed to `Synchronizer.createProposedTransactions`. It should never be called in
     /// production code.
     static func testOnlyFakeProposal(totalFee: UInt64) -> Self {
-        var ffiProposal = FfiProposal()
+        let ffiProposal = FfiProposal()
         var balance = FfiTransactionBalance()
 
         balance.feeRequired = totalFee

diff --git a/Sources/ZcashLightClientKit/Model/TransactionDataRequest.swift b/Sources/ZcashLightClientKit/Model/TransactionDataRequest.swift
@@ -0,0 +1,26 @@
+//
+//  TransactionDataRequest.swift
+//
+//
+//  Created by Lukáš Korba on 08-15-2024.
+//
+
+import Foundation
+
+struct SpendsFromAddress: Equatable {
+    let address: String
+    let blockRangeStart: UInt32
+    let blockRangeEnd: Int64
+}
+
+enum TransactionDataRequest: Equatable {
+    case getStatus([UInt8])
+    case enhancement([UInt8])
+    case spendsFromAddress(SpendsFromAddress)
+}
+
+enum TransactionStatus: Equatable {
+    case txidNotRecognized
+    case notInMainChain
+    case mined(BlockHeight)
+}
diff --git a/Sources/ZcashLightClientKit/Modules/Service/GRPC/LightWalletGRPCService.swift b/Sources/ZcashLightClientKit/Modules/Service/GRPC/LightWalletGRPCService.swift
@@ -193,7 +193,11 @@ extension LightWalletGRPCService: LightWalletService {
 
         do {
             let rawTx = try await compactTxStreamer.getTransaction(txFilter)
-            return ZcashTransaction.Fetched(rawID: txId, minedHeight: BlockHeight(rawTx.height), raw: rawTx.data)
+            return ZcashTransaction.Fetched(
+                rawID: txId,
+                minedHeight: rawTx.height == UInt64.max ? BlockHeight(-1) : BlockHeight(rawTx.height),
+                raw: rawTx.data
+            )
         } catch {
             let serviceError = error.mapToServiceError()
             throw ZcashError.serviceFetchTransactionFailed(serviceError)
@@ -280,6 +284,21 @@ extension LightWalletGRPCService: LightWalletService {
         try await compactTxStreamer.getTreeState(id)
     }
 
+    func getTaddressTxids(_ request: TransparentAddressBlockFilter) -> AsyncThrowingStream<RawTransaction, Error> {
+        let stream = compactTxStreamer.getTaddressTxids(request)
+        var iterator = stream.makeAsyncIterator()
+
+        return AsyncThrowingStream() {
+            do {
+                guard let rawTransaction = try await iterator.next() else { return nil }
+                return rawTransaction
+            } catch {
+                let serviceError = error.mapToServiceError()
+                throw ZcashError.serviceGetTaddressTxidsFailed(serviceError)
+            }
+        }
+    }
+
     func closeConnection() {
         _ = channel.close()
     }

diff --git a/Sources/ZcashLightClientKit/Modules/Service/LightWalletService.swift b/Sources/ZcashLightClientKit/Modules/Service/LightWalletService.swift
@@ -198,4 +198,6 @@ protocol LightWalletService: AnyObject {
     func getSubtreeRoots(_ request: GetSubtreeRootsArg) -> AsyncThrowingStream<SubtreeRoot, Error>
 
     func getTreeState(_ id: BlockID) async throws -> TreeState
+
+    func getTaddressTxids(_ request: TransparentAddressBlockFilter) -> AsyncThrowingStream<RawTransaction, Error>
 }