davide/PalladiumWallet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(spv): verify Merkle proofs progressively, gate spendability on it

Browse Source 
Balance/history now render as soon as tx downloads finish instead of
blocking on every historical Merkle proof, critical for mobile where
proof-checking can take much longer than the download itself. Proofs
continue to be checked in the background and each tx's Verified flag
catches up progressively; header ranges are now fetched in batches
(blockchain.block.headers) instead of one call per header to keep this
fast over high-latency links.

Coin selection (UtxoSpendability.IsSpendable) refuses to spend a UTXO
until its Merkle proof is actually checked, regardless of confirmation
count, so a server that fabricates a confirmed balance can get it
displayed early but never spent before the forgery is caught. The disk
cache only ever persists the fully-verified end state of a sync.

UI surfaces the new PendingVerificationSats/SpendableSats split with a
"verifying..." badge, and the sync save now runs off the UI thread to
avoid freezing on slower hardware.
This commit is contained in:
Davide Grilli
2026-07-17 19:41:31 +02:00
parent feb765663c
commit 3460e53b4f
16 changed files with 487 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/Spv/WalletSynchronizer.cs
+186 -46
View File
@@ -29,6 +29,23 @@ public sealed class SyncResult
/// <see cref="ConfirmedSats"/>.
/// </summary>
public required long ImmatureSats { get; init; }
/// <summary>
/// Confirmed and past its threshold, but not yet spendable because its Merkle proof
/// hasn't been checked yet (§7.4 progressive verification catching up in the background).
/// Subset of <see cref="ConfirmedSats"/> — NOT disjoint from <see cref="ImmatureSats"/>
/// (an immature coinbase can also be unverified), so never subtract both from
/// <see cref="ConfirmedSats"/> to get a spendable total — use <see cref="SpendableSats"/>.
/// </summary>
public required long PendingVerificationSats { get; init; }
/// <summary>
/// Sum of UTXOs that actually pass <see cref="Wallet.UtxoSpendability.IsSpendable"/> right
/// now — the true spendable balance. Computed directly from the same gate coin selection
/// uses, rather than by subtracting <see cref="ImmatureSats"/>/<see cref="PendingVerificationSats"/>
/// from <see cref="ConfirmedSats"/>, since those two can overlap.
/// </summary>
public required long SpendableSats { get; init; }
public required int NextReceiveIndex { get; init; }
public required int NextChangeIndex { get; init; }
public required IReadOnlyList<CachedTx> History { get; init; }
@@ -46,14 +63,37 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
/// <summary>Human-readable progress (for CLI and GUI status bar).</summary>
public event Action<string>? Progress;
/// <summary>
/// Fires with a fresh, self-consistent snapshot as soon as transaction downloads finish
/// (Merkle proofs may still be pending — see <see cref="CachedTx.Verified"/>/
/// <see cref="CachedUtxo.Verified"/>) and again periodically as background verification
/// progresses (§7.4). The wallet is usable after the first firing instead of waiting for
/// every historical proof to be checked; <see cref="SyncOnceAsync"/>'s returned Task still
/// only completes once verification is fully done, for callers that need the final state.
/// </summary>
public event Action<SyncResult>? PartialResult;
private readonly ConcurrentDictionary<string, Transaction> _txCache = new();
private readonly Dictionary<string, int> _verifiedAtHeight = [];
// Concurrent: written incrementally by individual merkle-verification tasks as they
// complete (§7.4 progressive verification), not just once after they all finish.
private readonly ConcurrentDictionary<string, int> _verifiedAtHeight = new();
private readonly ConcurrentDictionary<int, Task<string>> _headerFetches = new();
// checkpoint height -> highest height already proven to hash-chain back to it
// (in-memory only: cheap to recompute from _headerFetches, no need to persist).
private readonly ConcurrentDictionary<int, int> _anchoredUpTo = new();
// Serializes header-range downloads: concurrent AnchorToCheckpointAsync calls (one per tx
// being verified) would otherwise race on overlapping ranges and issue duplicate range
// requests. Fetches are network-bound and few (batches of up to 2016 headers), so
// serializing them costs nothing that matters.
private readonly SemaphoreSlim _headerRangeLock = new(1, 1);
// Max headers requested per blockchain.block.headers call. The server may return fewer
// (its own configured cap) — FetchHeaderRangeAsync loops on the actual count returned.
private const int HeaderBatchSize = 2016;
// Indices known from the previous sync: used by ScanChainAsync for incremental
// discovery — already-used addresses are fetched in a single burst instead of
// sequential batches, reducing round-trips from O(used/gapLimit) to O(1).
@@ -162,7 +202,15 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
foreach (var item in historyByAddress.Values.SelectMany(h => h))
txHeights[item.TxHash] = item.Height;
// 4+5. Download missing transactions and verify Merkle proofs in parallel.
// 4. Download missing transactions — needed to compute amounts/UTXOs locally.
// Merkle-proof verification (5) is deliberately NOT awaited together with this: on a
// wallet with thousands of transactions, downloads finish in seconds while proofs can
// take much longer over a high-latency link, and the wallet has everything it needs to
// show balance/history the moment downloads are done. Verification then continues in
// the background (§7.4 progressive verification), firing PartialResult as proofs land,
// while coin selection stays locked out of any UTXO until its own proof is checked
// (CachedUtxo.Verified, enforced in UtxoSpendability.IsSpendable) — a malicious server
// cannot get a fabricated balance spent just because it was shown early.
var network = PalladiumNetworks.For(account.Profile.Kind);
var missing = txHeights.Keys.Where(txid => !_txCache.ContainsKey(txid)).ToList();
var toVerify = txHeights
@@ -171,46 +219,81 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
.ToList();
if (missing.Count > 0 || toVerify.Count > 0)
{
Progress?.Invoke($"downloading {missing.Count} txs, verifying {toVerify.Count} proofs…");
var dlDone = 0;
var merkDone = 0;
var dlTasks = missing.Select(txid => RetryOnBusyAsync(async () =>
{
var raw = await client.GetTransactionAsync(txid, ct);
_txCache[txid] = Transaction.Parse(raw, network);
var n = Interlocked.Increment(ref dlDone);
if (n % 50 == 0 || n == missing.Count)
Progress?.Invoke($"tx {n}/{missing.Count}, proofs {merkDone}/{toVerify.Count}…");
}, ct));
var dlDone = 0;
await Task.WhenAll(missing.Select(txid => RetryOnBusyAsync(async () =>
{
var raw = await client.GetTransactionAsync(txid, ct);
_txCache[txid] = Transaction.Parse(raw, network);
var n = Interlocked.Increment(ref dlDone);
if (n % 50 == 0 || n == missing.Count)
Progress?.Invoke($"tx {n}/{missing.Count}, proofs 0/{toVerify.Count}…");
}, ct)));
var merkTasks = toVerify.Select(kv => RetryOnBusyAsync(async () =>
{
var (txid, height) = kv;
var proofTask = client.GetMerkleAsync(txid, height, ct);
var headerTask = _headerFetches.GetOrAdd(height,
h => client.GetBlockHeaderAsync(h, ct));
var proof = await proofTask;
var header = BlockHeaderInfo.Parse(await headerTask);
await AnchorToCheckpointAsync(height, ct);
if (!MerkleProof.Verify(
uint256.Parse(txid), proof.Pos,
proof.Merkle.Select(uint256.Parse), header.MerkleRoot))
throw new SpvVerificationException(
$"Invalid Merkle proof for {txid} (block {height}): server is not trustworthy.");
var n = Interlocked.Increment(ref merkDone);
if (n % 50 == 0 || n == toVerify.Count)
Progress?.Invoke($"tx {dlDone}/{missing.Count}, proofs {n}/{toVerify.Count}…");
}, ct));
SyncResult BuildSnapshot() =>
BuildResult(tip.Height, tracked, historyByAddress, txHeights, nextReceive, nextChange);
await Task.WhenAll(dlTasks.Concat(merkTasks));
foreach (var (txid, height) in toVerify)
_verifiedAtHeight[txid] = height;
}
PartialResult?.Invoke(BuildSnapshot());
var merkDone = 0;
var merkTasks = toVerify.Select(kv => RetryOnBusyAsync(async () =>
{
var (txid, height) = kv;
var proofTask = client.GetMerkleAsync(txid, height, ct);
// Anchor first: on a checkpointed height this fills _headerFetches[height]
// via the batched range fetch (§7.3), so the header lookup below is a cache
// hit instead of a second individual blockchain.block.header RPC per tx —
// halves round-trips for this stage on mainnet, where it matters most on
// high-latency mobile links. Falls back to an individual fetch when no
// checkpoint covers this height (testnet/regtest today).
await AnchorToCheckpointAsync(height, ct);
var headerHex = await _headerFetches.GetOrAdd(height, h => client.GetBlockHeaderAsync(h, ct));
var header = BlockHeaderInfo.Parse(headerHex);
var proof = await proofTask;
if (!MerkleProof.Verify(
uint256.Parse(txid), proof.Pos,
proof.Merkle.Select(uint256.Parse), header.MerkleRoot))
throw new SpvVerificationException(
$"Invalid Merkle proof for {txid} (block {height}): server is not trustworthy.");
_verifiedAtHeight[txid] = height;
var n = Interlocked.Increment(ref merkDone);
if (n % 50 == 0 || n == toVerify.Count)
Progress?.Invoke($"tx {missing.Count}/{missing.Count}, proofs {n}/{toVerify.Count}…");
if (n % PartialResultBatchSize == 0)
PartialResult?.Invoke(BuildSnapshot());
}, ct)).ToList();
if (merkTasks.Count > 0)
await Task.WhenAll(merkTasks);
return BuildSnapshot();
}
// Rebuilding the full snapshot (UTXOs/history/address rows) is O(wallet size); firing it on
// every single verified proof would make the background verification phase itself O(n²) for
// a wallet with thousands of transactions. Batching keeps "verified" badges catching up
// visibly without that cost.
private const int PartialResultBatchSize = 200;
private bool IsTxVerified(string txid, int height) =>
height <= 0 || (_verifiedAtHeight.TryGetValue(txid, out var vh) && vh == height);
/// <summary>
/// Assembles a <see cref="SyncResult"/> from the current state of <see cref="_txCache"/> and
/// <see cref="_verifiedAtHeight"/>. Callable multiple times per sync (§7.4): once as soon as
/// transaction downloads finish (proofs still pending), and again as verification progresses,
/// each time reflecting whichever transactions have been proof-checked so far.
/// </summary>
private SyncResult BuildResult(
int tipHeight,
List<TrackedAddress> tracked,
Dictionary<string, IReadOnlyList<HistoryItem>> historyByAddress,
Dictionary<string, int> txHeights,
int nextReceive,
int nextChange)
{
var transactions = txHeights.Keys.ToDictionary(txid => txid, txid => _txCache[txid]);
var verified = txHeights.ToDictionary(kv => kv.Key, kv => kv.Value > 0);
// 6. Local UTXO reconstruction.
var byScript = tracked.ToDictionary(t => t.ScriptPubKey, t => t);
@@ -222,6 +305,8 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
var utxos = new List<CachedUtxo>();
foreach (var (txid, tx) in transactions)
{
var height = txHeights[txid];
var verifiedTx = IsTxVerified(txid, height);
for (var vout = 0; vout < tx.Outputs.Count; vout++)
{
var output = tx.Outputs[vout];
@@ -237,8 +322,9 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
Address = addr.Address.ToString(),
IsChange = addr.IsChange,
AddressIndex = addr.Index,
Height = txHeights[txid],
Height = height,
IsCoinbase = tx.IsCoinBase,
Verified = verifiedTx,
});
}
}
@@ -247,6 +333,7 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
var history = new List<CachedTx>();
foreach (var (txid, tx) in transactions)
{
var height = txHeights[txid];
var received = tx.Outputs
.Where(o => byScript.ContainsKey(o.ScriptPubKey))
.Sum(o => o.Value.Satoshi);
@@ -257,9 +344,9 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
history.Add(new CachedTx
{
Txid = txid,
Height = txHeights[txid],
Height = height,
DeltaSats = received - sentSats,
Verified = verified[txid],
Verified = IsTxVerified(txid, height),
});
}
history.Sort((a, b) =>
@@ -286,12 +373,14 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
return new SyncResult
{
TipHeight = tip.Height,
TipHeight = tipHeight,
ConfirmedSats = utxos.Where(u => u.Height > 0).Sum(u => u.ValueSats),
UnconfirmedSats = utxos.Where(u => u.Height <= 0).Sum(u => u.ValueSats),
ImmatureSats = utxos.Where(u =>
u.Height > 0 && u.Confirmations(tip.Height) < u.RequiredConfirmations(account.Profile))
u.Height > 0 && u.Confirmations(tipHeight) < u.RequiredConfirmations(account.Profile))
.Sum(u => u.ValueSats),
PendingVerificationSats = utxos.Where(u => u.Height > 0 && !u.Verified).Sum(u => u.ValueSats),
SpendableSats = utxos.Where(u => u.IsSpendable(account.Profile, tipHeight)).Sum(u => u.ValueSats),
NextReceiveIndex = nextReceive,
NextChangeIndex = nextChange,
History = history,
@@ -325,9 +414,11 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
if (_anchoredUpTo.TryGetValue(cp.Height, out var anchoredTo) && anchoredTo >= height)
return;
var headers = await Task.WhenAll(Enumerable.Range(cp.Height, height - cp.Height + 1)
.Select(async h => BlockHeaderInfo.Parse(
await _headerFetches.GetOrAdd(h, hh => client.GetBlockHeaderAsync(hh, ct)))));
await FetchHeaderRangeAsync(cp.Height, height, ct);
var headers = Enumerable.Range(cp.Height, height - cp.Height + 1)
.Select(h => BlockHeaderInfo.Parse(_headerFetches[h].Result))
.ToArray();
if (!headers[0].MatchesCheckpoint(cp))
throw new SpvVerificationException(
@@ -341,6 +432,41 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
_anchoredUpTo.AddOrUpdate(cp.Height, height, (_, existing) => Math.Max(existing, height));
}
/// <summary>
/// Ensures every height in [<paramref name="fromHeight"/>, <paramref name="toHeightInclusive"/>]
/// is present in <see cref="_headerFetches"/>, downloading gaps with
/// blockchain.block.headers (§7.3) instead of one blockchain.block.header call per height.
/// </summary>
private async Task FetchHeaderRangeAsync(int fromHeight, int toHeightInclusive, CancellationToken ct)
{
await _headerRangeLock.WaitAsync(ct);
try
{
var h = fromHeight;
while (h <= toHeightInclusive)
{
if (_headerFetches.ContainsKey(h)) { h++; continue; }
var requested = Math.Min(HeaderBatchSize, toHeightInclusive - h + 1);
var range = await client.GetBlockHeadersAsync(h, requested, ct);
if (range.Count == 0)
throw new SpvVerificationException(
$"Server returned no headers starting at height {h}: server is not trustworthy.");
for (var i = 0; i < range.Count; i++)
{
var headerHex = range.Hex.Substring(i * BlockHeaderInfo.Size * 2, BlockHeaderInfo.Size * 2);
_headerFetches.TryAdd(h + i, Task.FromResult(headerHex));
}
h += range.Count;
}
}
finally
{
_headerRangeLock.Release();
}
}
/// <summary>
/// Scans one chain (receiving or change).
///
@@ -414,7 +540,12 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
return (firstUnused, tracked, history);
}
private static async Task RetryOnBusyAsync(Func<Task> op, CancellationToken ct)
// Counts "server busy" retries across the whole sync: surfaced via Progress so a slow
// sync can be diagnosed as server-side throttling (exponential backoff eating the time)
// rather than guessed at from wall-clock numbers alone.
private int _busyRetries;
private async Task RetryOnBusyAsync(Func<Task> op, CancellationToken ct)
{
var delay = 200;
for (var attempt = 0; ; attempt++)
@@ -423,13 +554,14 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
catch (ElectrumServerException ex)
when (IsBusy(ex) && attempt < 7)
{
ReportBusyRetry(delay);
await Task.Delay(delay, ct);
delay = Math.Min(delay * 2, 5_000);
}
}
}
private static async Task<T> RetryOnBusyAsync<T>(Func<Task<T>> op, CancellationToken ct)
private async Task<T> RetryOnBusyAsync<T>(Func<Task<T>> op, CancellationToken ct)
{
var delay = 200;
for (var attempt = 0; ; attempt++)
@@ -438,12 +570,20 @@ public sealed class WalletSynchronizer(IWalletAccount account, ElectrumClient cl
catch (ElectrumServerException ex)
when (IsBusy(ex) && attempt < 7)
{
ReportBusyRetry(delay);
await Task.Delay(delay, ct);
delay = Math.Min(delay * 2, 5_000);
}
}
}
private void ReportBusyRetry(int delayMs)
{
var n = Interlocked.Increment(ref _busyRetries);
if (n == 1 || n % 20 == 0)
Progress?.Invoke($"server busy, retry #{n} (waiting {delayMs}ms)…");
}
private static bool IsBusy(ElectrumServerException ex) =>
ex.Message.Contains("-102") ||
ex.Message.Contains("-101") ||