using System.Collections.Concurrent; using System.Threading; using NBitcoin; using PalladiumWallet.Core.Chain; using PalladiumWallet.Core.Crypto; using PalladiumWallet.Core.Net; using PalladiumWallet.Core.Storage; namespace PalladiumWallet.Core.Spv; /// Address derived and tracked during synchronisation. public sealed record TrackedAddress( BitcoinAddress Address, string ScriptHash, bool IsChange, int Index) { public Script ScriptPubKey => Address.ScriptPubKey; } /// Result of a synchronisation pass. public sealed class SyncResult { public required int TipHeight { get; init; } public required long ConfirmedSats { get; init; } public required long UnconfirmedSats { get; init; } public required int NextReceiveIndex { get; init; } public required int NextChangeIndex { get; init; } public required IReadOnlyList History { get; init; } public required IReadOnlyList Utxos { get; init; } public required IReadOnlyList Addresses { get; init; } public required IReadOnlyList AddressRows { get; init; } public required IReadOnlyDictionary Transactions { get; init; } } /// /// Wallet synchronisation (blueprint §7.4). /// public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient client, int gapLimit = 20) { /// Human-readable progress (for CLI and GUI status bar). public event Action? Progress; private readonly ConcurrentDictionary _txCache = new(); private readonly Dictionary _verifiedAtHeight = []; private readonly ConcurrentDictionary> _headerFetches = new(); // Indices known from the previous sync: used by ScanChainAsync for incremental // discovery — already-used addresses are fetched in a single burst instead of // sequential batches, reducing round-trips from O(used/gapLimit) to O(1). private int _knownReceiveIndex; private int _knownChangeIndex; /// /// Pre-populates internal caches from data saved on disk. /// Call before SyncOnceAsync to avoid re-downloading already known transactions. /// public void PreloadCaches( Dictionary rawTxHex, Dictionary verifiedAt, Dictionary? blockHeaders, int knownReceiveIndex, int knownChangeIndex, Network network) { foreach (var (txid, hex) in rawTxHex) _txCache.TryAdd(txid, Transaction.Parse(hex, network)); foreach (var (txid, height) in verifiedAt) if (!_verifiedAtHeight.ContainsKey(txid)) _verifiedAtHeight[txid] = height; if (blockHeaders is not null) foreach (var (height, hex) in blockHeaders) _headerFetches.TryAdd(height, Task.FromResult(hex)); _knownReceiveIndex = knownReceiveIndex; _knownChangeIndex = knownChangeIndex; } /// /// Exports the current caches in a serialisable form for disk storage. /// Only confirmed transactions (height > 0) are included: unconfirmed ones /// may change (RBF) and must always be re-downloaded. /// public (Dictionary RawTxHex, Dictionary VerifiedAt, Dictionary BlockHeaders) ExportCaches(Network network) { var rawHex = _verifiedAtHeight.Keys .Where(_txCache.ContainsKey) .ToDictionary(txid => txid, txid => _txCache[txid].ToHex()); // Only already-completed headers: in-progress Task instances // are not persisted (they will be re-fetched on the next sync if needed). var headers = new Dictionary(); foreach (var (height, task) in _headerFetches) if (task.IsCompletedSuccessfully) headers[height] = task.Result; return (rawHex, new Dictionary(_verifiedAtHeight), headers); } public async Task SyncOnceAsync(CancellationToken ct = default) { var tip = await client.SubscribeHeadersAsync(ct); Progress?.Invoke($"chain tip: {tip.Height}"); // 1-2. Address scanning. var tracked = new List(); var historyByAddress = new Dictionary>(); int nextReceive, nextChange; if (account.FixedAddresses is { } fixedAddresses) { foreach (var (addr, isChange, idx) in fixedAddresses) tracked.Add(new TrackedAddress(addr, Scripthash.FromAddress(addr), isChange, idx)); nextReceive = tracked.Count(t => !t.IsChange); nextChange = 0; await Task.WhenAll(tracked.Select(t => RetryOnBusyAsync(async () => { var h = await client.GetHistoryAsync(t.ScriptHash, ct); if (h.Count > 0) historyByAddress[t.ScriptHash] = h; }, ct)).Concat(tracked.Select(t => RetryOnBusyAsync(() => client.SubscribeScripthashAsync(t.ScriptHash, ct), ct)))); } else { // Receive and change chains in parallel (independent by definition). // ScanChainAsync uses _knownReceiveIndex/_knownChangeIndex for incremental // discovery: already-used addresses are fetched in a single burst. var receiveTask = ScanChainAsync(isChange: false, _knownReceiveIndex, ct); var changeTask = ScanChainAsync(isChange: true, _knownChangeIndex, ct); var rxScan = await receiveTask; var chScan = await changeTask; tracked.AddRange(rxScan.Tracked); tracked.AddRange(chScan.Tracked); foreach (var (k, v) in rxScan.History) historyByAddress[k] = v; foreach (var (k, v) in chScan.History) historyByAddress[k] = v; nextReceive = rxScan.NextIndex; nextChange = chScan.NextIndex; var gapAddresses = tracked.Where(t => (!t.IsChange && t.Index >= nextReceive && t.Index < nextReceive + gapLimit) || ( t.IsChange && t.Index >= nextChange && t.Index < nextChange + gapLimit)).ToList(); if (gapAddresses.Count > 0) await Task.WhenAll(gapAddresses.Select(t => RetryOnBusyAsync(() => client.SubscribeScripthashAsync(t.ScriptHash, ct), ct))); } // 3. Merged history (txid → highest reported height). var txHeights = new Dictionary(); foreach (var item in historyByAddress.Values.SelectMany(h => h)) txHeights[item.TxHash] = item.Height; // 4+5. Download missing transactions and verify Merkle proofs in parallel. var network = PalladiumNetworks.For(account.Profile.Kind); var missing = txHeights.Keys.Where(txid => !_txCache.ContainsKey(txid)).ToList(); var toVerify = txHeights .Where(kv => kv.Value > 0 && (!_verifiedAtHeight.TryGetValue(kv.Key, out var h) || h != kv.Value)) .ToList(); if (missing.Count > 0 || toVerify.Count > 0) { Progress?.Invoke($"downloading {missing.Count} txs, verifying {toVerify.Count} proofs…"); var dlDone = 0; var merkDone = 0; var dlTasks = missing.Select(txid => RetryOnBusyAsync(async () => { var raw = await client.GetTransactionAsync(txid, ct); _txCache[txid] = Transaction.Parse(raw, network); var n = Interlocked.Increment(ref dlDone); if (n % 50 == 0 || n == missing.Count) Progress?.Invoke($"tx {n}/{missing.Count}, proofs {merkDone}/{toVerify.Count}…"); }, ct)); var merkTasks = toVerify.Select(kv => RetryOnBusyAsync(async () => { var (txid, height) = kv; var proofTask = client.GetMerkleAsync(txid, height, ct); var headerTask = _headerFetches.GetOrAdd(height, h => client.GetBlockHeaderAsync(h, ct)); var proof = await proofTask; var header = BlockHeaderInfo.Parse(await headerTask); if (!MerkleProof.Verify( uint256.Parse(txid), proof.Pos, proof.Merkle.Select(uint256.Parse), header.MerkleRoot)) throw new SpvVerificationException( $"Invalid Merkle proof for {txid} (block {height}): server is not trustworthy."); var n = Interlocked.Increment(ref merkDone); if (n % 50 == 0 || n == toVerify.Count) Progress?.Invoke($"tx {dlDone}/{missing.Count}, proofs {n}/{toVerify.Count}…"); }, ct)); await Task.WhenAll(dlTasks.Concat(merkTasks)); foreach (var (txid, height) in toVerify) _verifiedAtHeight[txid] = height; } var transactions = txHeights.Keys.ToDictionary(txid => txid, txid => _txCache[txid]); var verified = txHeights.ToDictionary(kv => kv.Key, kv => kv.Value > 0); // 6. Local UTXO reconstruction. var byScript = tracked.ToDictionary(t => t.ScriptPubKey, t => t); var spent = transactions.Values .SelectMany(tx => tx.Inputs) .Select(i => i.PrevOut) .ToHashSet(); var utxos = new List(); foreach (var (txid, tx) in transactions) { for (var vout = 0; vout < tx.Outputs.Count; vout++) { var output = tx.Outputs[vout]; if (!byScript.TryGetValue(output.ScriptPubKey, out var addr)) continue; if (spent.Contains(new OutPoint(tx, vout))) continue; utxos.Add(new CachedUtxo { Txid = txid, Vout = vout, ValueSats = output.Value.Satoshi, Address = addr.Address.ToString(), IsChange = addr.IsChange, AddressIndex = addr.Index, Height = txHeights[txid], }); } } // 7. Delta per history entry. var history = new List(); foreach (var (txid, tx) in transactions) { var received = tx.Outputs .Where(o => byScript.ContainsKey(o.ScriptPubKey)) .Sum(o => o.Value.Satoshi); var sentSats = tx.Inputs .Where(i => transactions.TryGetValue(i.PrevOut.Hash.ToString(), out var prev) && byScript.ContainsKey(prev.Outputs[i.PrevOut.N].ScriptPubKey)) .Sum(i => transactions[i.PrevOut.Hash.ToString()].Outputs[i.PrevOut.N].Value.Satoshi); history.Add(new CachedTx { Txid = txid, Height = txHeights[txid], DeltaSats = received - sentSats, Verified = verified[txid], }); } history.Sort((a, b) => { var ha = a.Height <= 0 ? int.MaxValue : a.Height; var hb = b.Height <= 0 ? int.MaxValue : b.Height; return hb.CompareTo(ha); }); var balanceByAddress = utxos .GroupBy(u => u.Address) .ToDictionary(g => g.Key, g => g.Sum(u => u.ValueSats)); var addressRows = tracked .OrderBy(t => t.IsChange).ThenBy(t => t.Index) .Select(t => new CachedAddress { Address = t.Address.ToString(), IsChange = t.IsChange, Index = t.Index, BalanceSats = balanceByAddress.GetValueOrDefault(t.Address.ToString()), TxCount = historyByAddress.TryGetValue(t.ScriptHash, out var h) ? h.Count : 0, }) .ToList(); return new SyncResult { TipHeight = tip.Height, ConfirmedSats = utxos.Where(u => u.Height > 0).Sum(u => u.ValueSats), UnconfirmedSats = utxos.Where(u => u.Height <= 0).Sum(u => u.ValueSats), NextReceiveIndex = nextReceive, NextChangeIndex = nextChange, History = history, Utxos = utxos, Addresses = tracked, AddressRows = addressRows, Transactions = transactions, }; } /// /// Scans one chain (receiving or change). /// /// Phase 1 — known addresses (0..fromIndex-1): all GetHistoryAsync calls are /// fired in a single parallel burst, with no sequential batching. A wallet /// with 100 used addresses costs 1 RTT instead of 5 sequential gap-limit rounds. /// /// Phase 2 — discovery from fromIndex onwards: gap-limit batching as before, /// required to know when to stop. /// private async Task<(int NextIndex, List Tracked, Dictionary> History)> ScanChainAsync(bool isChange, int fromIndex, CancellationToken ct) { var tracked = new List(); var history = new Dictionary>(); // Phase 1: single burst for all already-known addresses. if (fromIndex > 0) { var known = Enumerable.Range(0, fromIndex).Select(i => { var addr = account.GetAddress(isChange, i); return new TrackedAddress(addr, Scripthash.FromAddress(addr), isChange, i); }).ToList(); tracked.AddRange(known); var knownHistories = await Task.WhenAll( known.Select(t => RetryOnBusyAsync( () => client.GetHistoryAsync(t.ScriptHash, ct), ct))); for (var i = 0; i < known.Count; i++) if (knownHistories[i].Count > 0) history[known[i].ScriptHash] = knownHistories[i]; } // Phase 2: gap-limit discovery from fromIndex onwards. var consecutiveEmpty = 0; var index = fromIndex; var firstUnused = fromIndex; while (consecutiveEmpty < gapLimit) { var batch = Enumerable.Range(index, gapLimit).Select(i => { var addr = account.GetAddress(isChange, i); return new TrackedAddress(addr, Scripthash.FromAddress(addr), isChange, i); }).ToList(); index += batch.Count; tracked.AddRange(batch); var histories = await Task.WhenAll( batch.Select(t => RetryOnBusyAsync( () => client.GetHistoryAsync(t.ScriptHash, ct), ct))); for (var i = 0; i < batch.Count && consecutiveEmpty < gapLimit; i++) { if (histories[i].Count == 0) { consecutiveEmpty++; } else { consecutiveEmpty = 0; firstUnused = batch[i].Index + 1; history[batch[i].ScriptHash] = histories[i]; } } } return (firstUnused, tracked, history); } private static async Task RetryOnBusyAsync(Func op, CancellationToken ct) { var delay = 200; for (var attempt = 0; ; attempt++) { try { await op(); return; } catch (ElectrumServerException ex) when (IsBusy(ex) && attempt < 7) { await Task.Delay(delay, ct); delay = Math.Min(delay * 2, 5_000); } } } private static async Task RetryOnBusyAsync(Func> op, CancellationToken ct) { var delay = 200; for (var attempt = 0; ; attempt++) { try { return await op(); } catch (ElectrumServerException ex) when (IsBusy(ex) && attempt < 7) { await Task.Delay(delay, ct); delay = Math.Min(delay * 2, 5_000); } } } private static bool IsBusy(ElectrumServerException ex) => ex.Message.Contains("-102") || ex.Message.Contains("-101") || ex.Message.Contains("server busy", StringComparison.OrdinalIgnoreCase) || ex.Message.Contains("excessive resource usage", StringComparison.OrdinalIgnoreCase); } /// SPV verification failed: server data contradicts the proofs (§17). public sealed class SpvVerificationException(string message) : Exception(message);