# Copyright (c) 2016-2018, Neil Booth # # All rights reserved. # # See the file "LICENCE" for information about the copyright # and warranty status of this software. '''Mempool handling.''' import itertools import time from abc import ABC, abstractmethod import asyncio from collections import defaultdict from dataclasses import dataclass from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set, Iterable import math from graphlib import TopologicalSorter from aiorpcx import run_in_thread, sleep, ignore_after from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash from electrumx.lib.tx import SkipTxDeserialize from electrumx.lib.util import class_logger, chunks, OldTaskGroup, LogTimeTaken from electrumx.lib.tx import TXOSpendStatus, TxOutpoint from electrumx.server.db import UTXO if TYPE_CHECKING: from electrumx.lib.coins import Coin DB_UTXO_MAP = dict[TxOutpoint, Optional[tuple[bytes, int]]] # prevout->(hashX,value_in_sats) @dataclass(slots=True) class MemPoolTx: prevouts: Sequence[TxOutpoint] # (txid_rev, txout_idx) # A pair is a (hashX, value) tuple in_pairs: Optional[Sequence[tuple[bytes, int]]] # (hashX, value_in_sats) out_pairs: Sequence[tuple[bytes, int]] # (hashX, value_in_sats) fee: int # in sats size: int # in vbytes @dataclass(slots=True) class MemPoolTxSummary: txid_rev: bytes fee: int # in sats has_unconfirmed_inputs: bool @dataclass(slots=True, frozen=True, kw_only=True) class RecentMemPoolTx: txid_rev: bytes fee: int # in sats vsize: int # in vbytes class DBSyncError(Exception): pass class MemPoolAPI(ABC): '''A concrete instance of this class is passed to the MemPool object and used by it to query DB and blockchain state.''' @abstractmethod async def height(self) -> int: '''Query bitcoind for its height.''' @abstractmethod def cached_height(self) -> Optional[int]: '''Return the height of bitcoind the last time it was queried, for any reason, without actually querying it. ''' @abstractmethod async def daemon_height_changed(self) -> None: '''Wait until bitcoind's height changes.''' @abstractmethod def db_height(self) -> int: '''Return the height flushed to the on-disk DB.''' @abstractmethod async def db_height_changed(self) -> None: '''Wait until the on-disk DB height changes.''' @abstractmethod async def mempool_txids_hum(self) -> Sequence[str]: '''Query bitcoind for the txids of all transactions in its mempool, returned as a list.''' @abstractmethod async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]: '''Query bitcoind for the serialized raw transactions with the given txids. Missing transactions are returned as None. txids_hum is an iterable of hexadecimal hash strings.''' @abstractmethod async def lookup_utxos(self, prevouts: Sequence[TxOutpoint]) -> Sequence[Optional[Tuple[bytes, int]]]: '''Return a list of (hashX, value) pairs, one for each prevout if unspent, otherwise return None if spent or not found (for the given prevout). prevouts - an iterable of (txid_rev, txout_idx) pairs ''' @abstractmethod async def on_mempool( self, *, touched_hashxs: Set[bytes], touched_outpoints: Set[TxOutpoint], height: int, ): '''Called each time the mempool is synchronized. touched_hashxs and touched_outpoints are sets of hashXs and tx outpoints touched since the previous call. height is the daemon's height at the time the mempool was obtained. ''' class MemPool: '''Representation of the daemon's mempool. coin - a coin class from coins.py api - an object implementing MemPoolAPI Updated regularly in caught-up state. Goal is to enable efficient response to the calls in the external interface. To that end we maintain the following maps: tx: txid_rev -> MemPoolTx hashXs: hashX -> set of all txids_rev of txs touching the hashX ''' def __init__( self, coin: Type['Coin'], api: MemPoolAPI, *, refresh_secs=5.0, log_status_secs=60.0, ): assert isinstance(api, MemPoolAPI) self.coin = coin self.api = api self.logger = class_logger(__name__, self.__class__.__name__) self.txs = {} # type: Dict[bytes, MemPoolTx] # txid_rev->tx self.hashXs = defaultdict(set) # type: Dict[Optional[bytes], Set[bytes]] # hashX->txids_rev self.txo_to_spender = {} # type: Dict[TxOutpoint, bytes] # prevout->txid_rev self.cached_compact_histogram = [] # type: Sequence[tuple[float, int]] self.refresh_secs = refresh_secs self.log_status_secs = log_status_secs # Prevents mempool refreshes during fee histogram calculation self.lock = asyncio.Lock() async def _logging(self, synchronized_event): '''Print regular logs of mempool stats.''' self.logger.info('beginning processing of daemon mempool. ' 'This can take some time...') start = time.monotonic() await synchronized_event.wait() elapsed = time.monotonic() - start self.logger.info(f'synced in {elapsed:.2f}s') while True: mempool_size = sum(tx.size for tx in self.txs.values()) / 1_000_000 self.logger.info(f'{len(self.txs):,d} txs {mempool_size:.2f} MB, ' f'touching {len(self.hashXs):,d} addresses. ' f'{len(self.txo_to_spender):,d} spends.') await sleep(self.log_status_secs) await synchronized_event.wait() async def _refresh_histogram(self, synchronized_event): while True: await synchronized_event.wait() async with self.lock: # Threaded as can be expensive bin_size = self.coin.MEMPOOL_COMPACT_HISTOGRAM_BINSIZE await run_in_thread(self._update_histogram, bin_size) await sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS) def _update_histogram(self, bin_size): # Build a histogram by fee rate t0 = time.monotonic() histogram = defaultdict(int) for tx in self.txs.values(): fee_rate = tx.fee / tx.size # use 0.1 sat/byte resolution # note: rounding *down* is intentional. This ensures txs # with a given fee rate will end up counted in the expected # bucket/interval of the compact histogram. fee_rate = math.floor(10 * fee_rate) / 10 histogram[fee_rate] += tx.size compact = self._compress_histogram(histogram, bin_size=bin_size) self.logger.info(f'compact fee histogram: {compact}. took {time.monotonic() - t0:.3f}s.') self.cached_compact_histogram = compact @classmethod def _compress_histogram( cls, histogram: Dict[float, int], *, bin_size: int ) -> Sequence[Tuple[float, int]]: '''Calculate and return a compact fee histogram as needed for "mempool.get_fee_histogram" protocol request. histogram: feerate (sat/vbyte) -> total size in bytes of txs that pay approx feerate bin_size: ~minimum vsize of a bucket of txs in the result (e.g. 100 kb) ''' # Now compact it. For efficiency, get_fees returns a # compact histogram with variable bin size. The compact # histogram is an array of (fee_rate, vsize) values. # vsize_n is the cumulative virtual size of mempool # transactions with a fee rate in the interval # [rate_(n-1), rate_n)], and rate_(n-1) > rate_n. # Intervals are chosen to create tranches containing at # least a certain cumulative size (bin_size) of transactions. assert bin_size > 0 compact = [] cum_size = 0 prev_fee_rate = None for fee_rate, size in sorted(histogram.items(), reverse=True): # if there is a big lump of txns at this specific size, # consider adding the previous item now (if not added already) if size > 2 * bin_size and prev_fee_rate is not None and cum_size > 0: compact.append((prev_fee_rate, cum_size)) cum_size = 0 bin_size *= 1.1 # now consider adding this item cum_size += size if cum_size > bin_size: compact.append((fee_rate, cum_size)) cum_size = 0 bin_size *= 1.1 prev_fee_rate = fee_rate return compact def _accept_transactions( self, *, tx_map: Dict[bytes, MemPoolTx], # txid_rev->tx utxo_map: DB_UTXO_MAP, # prevout->(hashX,value_in_sats) touched_hashxs: Set[bytes], # set of hashXs touched_outpoints: Set[TxOutpoint], # set of outpoints topologically_sort: bool, ) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]: '''Accept transactions in tx_map to the mempool if all their inputs can be found in the existing mempool or a utxo_map from the DB. Returns an (unprocessed tx_map, unspent utxo_map) pair. ''' hashXs = self.hashXs txs = self.txs txo_to_spender = self.txo_to_spender if topologically_sort: # Sort tx_map so that parent txs come first (in case of unconf chain). # This is just an optimization so that fewer txs will get "deferred", leading to better perf. # Dependencies outside tx_map (already in chainstate (utxo_map), or our mempool (self.txs), # or MISSING) are ignored. cand_to_parents = { txid: {parent_txid for (parent_txid, txout_idx) in tx.prevouts if parent_txid in tx_map} for (txid, tx) in tx_map.items()} cand_txids = list(TopologicalSorter(cand_to_parents).static_order()) assert len(tx_map) == len(cand_to_parents) == len(cand_txids), \ f"{len(tx_map)=}, {len(cand_to_parents)=}, {len(cand_txids)=}" del cand_to_parents else: cand_txids = tx_map.keys() deferred = {} # type: dict[bytes, MemPoolTx] unspent = set(utxo_map) # Try to find all prevouts so we can accept the candidate TXs from tx_map into our mempool for txid_rev in cand_txids: tx = tx_map[txid_rev] in_pairs = [] try: for prevout in tx.prevouts: # first, look for parent tx among confirmed UTXOs: utxo = utxo_map.get(prevout) if not utxo: # second, look for parent tx in mempool prev_hash, prev_index = prevout # Raises KeyError if prev_hash is not in txs utxo = txs[prev_hash].out_pairs[prev_index] in_pairs.append(utxo) except KeyError: deferred[txid_rev] = tx continue # Spend the prevouts unspent.difference_update(tx.prevouts) # Save the in_pairs, compute the fee and accept the TX tx.in_pairs = tuple(in_pairs) # Avoid negative fees if dealing with generation-like transactions # because some in_parts would be missing tx.fee = max(0, (sum(v for _, v in tx.in_pairs) - sum(v for _, v in tx.out_pairs))) txs[txid_rev] = tx for hashX, _value in itertools.chain(tx.in_pairs, tx.out_pairs): touched_hashxs.add(hashX) hashXs[hashX].add(txid_rev) for prevout in tx.prevouts: txo_to_spender[prevout] = txid_rev touched_outpoints.add(prevout) for out_idx, out_pair in enumerate(tx.out_pairs): touched_outpoints.add((txid_rev, out_idx)) return deferred, {prevout: utxo_map[prevout] for prevout in unspent} async def _refresh_hashes(self, synchronized_event): '''Refresh our view of the daemon's mempool.''' # touched_* accumulates between calls to on_mempool and each # call transfers ownership touched_hashxs = set() touched_outpoints = set() prev_mempool_height = -1 while True: height = self.api.cached_height() is_new_block = height != prev_mempool_height with LogTimeTaken(self.logger, "getrawmempool RPC", enabled=is_new_block): txids_hum = await self.api.mempool_txids_hum() if height != await self.api.height(): # if height changed *again*, re-start continue with LogTimeTaken(self.logger, "convert_txids", enabled=is_new_block): txids_rev = await run_in_thread(lambda: {hex_str_to_hash(hh) for hh in txids_hum}) if self.api.db_height() < height: # DB not yet caught up. give it some time... async with ignore_after(self.refresh_secs): await self.api.db_height_changed() try: async with self.lock: await self._process_mempool( all_txids_rev=txids_rev, touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, mempool_height=height, debug_logs=is_new_block, ) except DBSyncError: # The UTXO DB is not at the same height as the # mempool; wait and try again self.logger.debug('waiting for DB to sync') else: synchronized_event.set() synchronized_event.clear() await self.api.on_mempool( touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, height=height, ) prev_mempool_height = height touched_hashxs = set() touched_outpoints = set() # poll bitcoind's whole mempool every few seconds; or instantly after the DB height changes async with ignore_after(self.refresh_secs): async with OldTaskGroup(wait=any) as group: await group.spawn(self.api.db_height_changed()) await group.spawn(self.api.daemon_height_changed()) async def _process_mempool( self, *, all_txids_rev: Set[bytes], # set of txids_rev # complete view of daemon's mempool touched_hashxs: Set[bytes], # set of hashXs touched_outpoints: Set[TxOutpoint], # set of outpoints mempool_height: int, debug_logs: bool = False, ) -> None: # Re-sync with the new set of hashes txs = self.txs hashXs = self.hashXs txo_to_spender = self.txo_to_spender if mempool_height != self.api.db_height(): # FIXME should compare blockhash raise DBSyncError # 1. Handle txs that have disappeared (evicted, just got mined, etc) # TODO split disappeared txs workload into a threadpool, chunks of ~200 txs def handle_disappeared_txs() -> None: nonlocal touched_hashxs for txid_rev in disappeared_hashes: tx = txs.pop(txid_rev) # hashXs tx_hashXs = {hashX for hashX, value in tx.in_pairs} tx_hashXs.update(hashX for hashX, value in tx.out_pairs) for hashX in tx_hashXs: hashXs[hashX].remove(txid_rev) if not hashXs[hashX]: del hashXs[hashX] touched_hashxs |= tx_hashXs # outpoints for prevout in tx.prevouts: del txo_to_spender[prevout] touched_outpoints.add(prevout) for out_idx, out_pair in enumerate(tx.out_pairs): touched_outpoints.add((txid_rev, out_idx)) with LogTimeTaken(self.logger, "_process_mempool() removing txs", enabled=debug_logs): disappeared_hashes = await run_in_thread(lambda: set(txs) - all_txids_rev) handle_disappeared_txs() # 2. Process new transactions t0 = time.monotonic() new_hashes = await run_in_thread(lambda: list(all_txids_rev.difference(txs))) if new_hashes: # 2.1. fetch raw txs from bitcoin daemon group = OldTaskGroup() for hashes in chunks(new_hashes, 200): coro = self._fetch_raw_txs_and_utxos( new_txids_rev=hashes, all_txids_rev=all_txids_rev, ) await group.spawn(coro) if mempool_height != self.api.db_height(): raise DBSyncError tx_map = {} # type: dict[bytes, MemPoolTx] utxo_map = {} # type: DB_UTXO_MAP async for task in group: partial_tx_map, partial_utxo_map = task.result() tx_map.update(partial_tx_map) utxo_map.update(partial_utxo_map) if debug_logs: self.logger.debug(f'_process_mempool() fetching txs took {time.monotonic() - t0:.3f}s. {len(new_hashes)=}') # 2.2. accept txs into our mempool def accept_txs_loop() -> None: # Accept candidate txs from tx_map into our mempool. # We only accept candidates for which we can find a UTXO for each tx input: # - some UTXOs we find in the DB=utxo_map (chainstate), # - some we find in self.txs (our memool), # - some we might only find in tx_map (among the candidates: consider long chain of unconfirmed txs) # - some we might not find anywhere: if DB is corrupted or bitcoind gave inconsistent mempool, or other race # In each iteration, we loop over tx_map and move accepted transactions from it to self.txs. # In the worst degenerate case, this could be O(n^2). To avoid that, we topologically sort tx_map. nonlocal tx_map, utxo_map prior_txmap_size = 0 while tx_map and len(tx_map) != prior_txmap_size: prior_txmap_size = len(tx_map) tx_map, utxo_map = self._accept_transactions( tx_map=tx_map, utxo_map=utxo_map, touched_hashxs=touched_hashxs, touched_outpoints=touched_outpoints, topologically_sort=True, ) if tx_map: self.logger.error(f'{len(tx_map)} txs dropped') with LogTimeTaken(self.logger, f"_process_mempool() accepting txs ({len(new_hashes)=})", enabled=debug_logs): accept_txs_loop() async def _fetch_raw_txs_and_utxos( self, *, new_txids_rev: set[bytes], # (some) new candidate txs for our mempool all_txids_rev: set[bytes], # complete view of daemon's mempool ) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]: '''Fetch a list of mempool transactions, and lookup corresponding UTXOs in the DB.''' txids_hum_iter = (hash_to_hex_str(hash) for hash in new_txids_rev) raw_txs = await self.api.raw_transactions(txids_hum_iter) def deserialize_txs() -> Dict[bytes, MemPoolTx]: """This function is pure""" to_hashX = self.coin.hashX_from_script deserializer = self.coin.DESERIALIZER txs = {} # type: Dict[bytes, MemPoolTx] for txid_rev, raw_tx in zip(new_txids_rev, raw_txs): # The daemon may have evicted the tx from its # mempool or it may have gotten in a block if not raw_tx: continue try: tx, tx_size = deserializer(raw_tx).read_tx_and_vsize() except SkipTxDeserialize as ex: self.logger.debug(f'skipping tx {hash_to_hex_str(txid_rev)}: {ex}') continue # Convert the inputs and outputs into (hashX, value) pairs # Drop generation-like inputs from MemPoolTx.prevouts txin_pairs = tuple((txin.prev_txid_rev, txin.prev_idx) for txin in tx.inputs if not txin.is_generation()) txout_pairs = tuple((to_hashX(txout.pk_script), txout.value) for txout in tx.outputs) txs[txid_rev] = MemPoolTx( prevouts=txin_pairs, in_pairs=None, out_pairs=txout_pairs, fee=0, size=tx_size, ) return txs # Thread this potentially slow operation so as not to block tx_map = await run_in_thread(deserialize_txs) # type: Dict[bytes, MemPoolTx] # Determine all prevouts not in the mempool, and fetch the # UTXO information from the database. Failed prevout lookups # return None - concurrent database updates happen - which is # relied upon by _accept_transactions. Ignore prevouts that are # generation-like. prevouts = tuple(prevout for tx in tx_map.values() for prevout in tx.prevouts if prevout[0] not in all_txids_rev) utxos = await self.api.lookup_utxos(prevouts) utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)} return tx_map, utxo_map # # External interface # async def keep_synchronized(self, synchronized_event: asyncio.Event) -> None: '''Keep the mempool synchronized with the daemon.''' async with OldTaskGroup() as group: await group.spawn(self._refresh_hashes(synchronized_event)) await group.spawn(self._refresh_histogram(synchronized_event)) await group.spawn(self._logging(synchronized_event)) async def balance_delta(self, hashX: bytes) -> int: '''Return the unconfirmed amount in the mempool for hashX. Can be positive or negative. ''' value = 0 if hashX in self.hashXs: for txid_rev in self.hashXs[hashX]: tx = self.txs[txid_rev] value -= sum(v for h168, v in tx.in_pairs if h168 == hashX) value += sum(v for h168, v in tx.out_pairs if h168 == hashX) return value async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]: '''Return a compact fee histogram of the current mempool.''' return self.cached_compact_histogram async def potential_spends(self, hashX: bytes) -> set[TxOutpoint]: '''Return a set of (prev_hash, prev_idx) pairs from mempool transactions that touch hashX. None, some or all of these may be spends of the hashX, but all actual spends of it (in the DB or mempool) will be included. ''' result = set() for txid_rev in self.hashXs.get(hashX, ()): tx = self.txs[txid_rev] result.update(tx.prevouts) return result async def transaction_summaries(self, hashX: bytes) -> Sequence[MemPoolTxSummary]: '''Return a list of MemPoolTxSummary objects for the hashX, sorted as expected by protocol methods. ''' result = [] # type: list[MemPoolTxSummary] for txid_rev in self.hashXs.get(hashX, ()): tx = self.txs[txid_rev] has_ui = any(hash in self.txs for hash, idx in tx.prevouts) result.append(MemPoolTxSummary(txid_rev, tx.fee, has_ui)) result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.txid_rev[::-1])) return result async def unordered_UTXOs(self, hashX: bytes) -> Sequence[UTXO]: '''Return an unordered list of UTXO named tuples from mempool transactions that pay to hashX. This does not consider if any other mempool transactions spend the outputs. ''' utxos = [] for txid_rev in self.hashXs.get(hashX, ()): tx = self.txs.get(txid_rev) for pos, (hX, value) in enumerate(tx.out_pairs): if hX == hashX: utxos.append(UTXO(-1, pos, txid_rev, 0, value)) return utxos async def spender_for_txo(self, prev_txid_rev: bytes, txout_idx: int) -> 'TXOSpendStatus': '''For an outpoint, returns its spend-status. This only considers the mempool, not the DB/blockchain, so e.g. mined txs are not distinguished from txs that never existed. ''' # look up funding tx prev_tx = self.txs.get(prev_txid_rev, None) if prev_tx is None: # funding tx already mined or never existed funder_height = None else: if len(prev_tx.out_pairs) <= txout_idx: # output idx out of bounds...? return TXOSpendStatus(funder_height=None) prev_has_ui = any(hash in self.txs for hash, idx in prev_tx.prevouts) funder_height = -prev_has_ui prevout = (prev_txid_rev, txout_idx) # look up spending tx spender_txid_rev = self.txo_to_spender.get(prevout, None) if spender_txid_rev is None: return TXOSpendStatus(funder_height=funder_height) spender_tx = self.txs.get(spender_txid_rev, None) if spender_tx is None: self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txid_rev)} not in" f"mempool, but txo_to_spender referenced it as spender " f"of {hash_to_hex_str(prev_txid_rev)}:{txout_idx} ?!") return TXOSpendStatus(funder_height=funder_height) spender_has_ui = any(txid_rev in self.txs for txid_rev, idx in spender_tx.prevouts) spender_height = -spender_has_ui return TXOSpendStatus( funder_height=funder_height, spender_txid_rev=spender_txid_rev, spender_height=spender_height, ) async def get_recently_added_txs(self, *, count: int) -> Sequence[RecentMemPoolTx]: # note: inefficient for large "count"s it = reversed(self.txs.items()) count = min(count, len(self.txs)) mempool_txs = [next(it) for _ in range(count)] return [ RecentMemPoolTx(txid_rev=hash, fee=mtx.fee, vsize=mtx.size) for hash, mtx in mempool_txs]