Initial commit: snapshot of spesmilo/electrumx 2.0.0

Unmodified source tree of upstream commit 280cb339aa158e099eb936033eecd15719770651
("Merge pull request #389 from SomberNight/202607_release_2_0",
https://github.com/spesmilo/electrumx, release 2.0.0, 3 July 2026).

All Palladium-specific changes are in the commits that follow.

src/electrumx/server/controller.py

@@ -0,0 +1,199 @@
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+from asyncio import Event
+from typing import Set, Dict, Tuple
+
+from aiorpcx import _version as aiorpcx_version
+
+import electrumx
+from electrumx.lib.server_base import ServerBase
+from electrumx.lib.util import version_string, OldTaskGroup
+from electrumx.lib.tx import TxOutpoint
+from electrumx.server.db import DB
+from electrumx.server.mempool import MemPool, MemPoolAPI
+from electrumx.server.session import SessionManager
+
+
+class Notifications:
+    # hashX notifications come from two sources: new blocks and
+    # mempool refreshes.
+    #
+    # A user with a pending transaction is notified after the block it
+    # gets in is processed.  Block processing can take an extended
+    # time, and the prefetcher might poll the daemon after the mempool
+    # code in any case.  In such cases the transaction will not be in
+    # the mempool after the mempool refresh.  We want to avoid
+    # notifying clients twice - for the mempool refresh and when the
+    # block is done.  This object handles that logic by deferring
+    # notifications appropriately.
+
+    def __init__(self):
+        self._touched_hashxs_mp = {}  # type: Dict[int, Set[bytes]]
+        self._touched_hashxs_bp = {}  # type: Dict[int, Set[bytes]]
+        self._touched_outpoints_mp = {}  # type: Dict[int, Set[TxOutpoint]]
+        self._touched_outpoints_bp = {}  # type: Dict[int, Set[TxOutpoint]]
+        self._highest_block = -1
+
+    async def _maybe_notify(self):
+        th_mp, th_bp = self._touched_hashxs_mp, self._touched_hashxs_bp
+        # figure out block height
+        common_heights = set(th_mp).intersection(th_bp)
+        if common_heights:
+            height = max(common_heights)
+        elif th_mp and max(th_mp) == self._highest_block:
+            height = self._highest_block
+        else:
+            # Either we are processing a block and waiting for it to
+            # come in, or we have not yet had a mempool update for the
+            # new block height
+            return
+        # hashXs
+        touched_hashxs = th_mp.pop(height)
+        for old in [h for h in th_mp if h <= height]:
+            del th_mp[old]
+        for old in [h for h in th_bp if h <= height]:
+            touched_hashxs.update(th_bp.pop(old))
+        # outpoints
+        to_mp, to_bp = self._touched_outpoints_mp, self._touched_outpoints_bp
+        touched_outpoints = to_mp.pop(height)
+        for old in [h for h in to_mp if h <= height]:
+            del to_mp[old]
+        for old in [h for h in to_bp if h <= height]:
+            touched_outpoints.update(to_bp.pop(old))
+
+        await self.notify(
+            height=height,
+            touched_hashxs=touched_hashxs,
+            touched_outpoints=touched_outpoints,
+        )
+
+    async def notify(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        pass
+
+    async def start(self, height: int, notify_func):
+        self._highest_block = height
+        self.notify = notify_func
+        await self.notify(
+            height=height,
+            touched_hashxs=set(),
+            touched_outpoints=set(),
+        )
+
+    async def on_mempool(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        self._touched_hashxs_mp[height] = touched_hashxs
+        self._touched_outpoints_mp[height] = touched_outpoints
+        await self._maybe_notify()
+
+    async def on_block(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        self._touched_hashxs_bp[height] = touched_hashxs
+        self._touched_outpoints_bp[height] = touched_outpoints
+        self._highest_block = height
+        await self._maybe_notify()
+
+
+class Controller(ServerBase):
+    '''Manages server initialisation and shutdown.
+
+    Servers are started once the mempool is synced after the block
+    processor first catches up with the daemon.
+    '''
+    async def serve(self, shutdown_event):
+        '''Start the RPC server and wait for the mempool to synchronize.  Then
+        start serving external clients.
+        '''
+        if not (0, 25, 0) <= aiorpcx_version < (0, 26):
+            raise RuntimeError(
+                f'aiorpcX version {aiorpcx_version!r} does not match required: 0.25.0<=ver<0.26')
+
+        env = self.env
+        min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
+        self.logger.info(f'software version: {electrumx.version}')
+        self.logger.info(f'aiorpcX version: {version_string(aiorpcx_version)}')
+        self.logger.info(f'supported protocol versions: {min_str}-{max_str}')
+        self.logger.info(f'event loop policy: {env.loop_policy}')
+        self.logger.info(f'reorg limit is {env.reorg_limit:,d} blocks')
+
+        notifications = Notifications()
+        Daemon = env.coin.DAEMON
+        BlockProcessor = env.coin.BLOCK_PROCESSOR
+
+        async with Daemon(env.coin, env.daemon_url) as daemon:
+            db = DB(env)
+            bp = BlockProcessor(env, db, daemon, notifications)
+
+            # Set notifications up to implement the MemPoolAPI
+            def get_db_height():
+                return db.db_height
+            notifications.height = daemon.height
+            notifications.db_height = get_db_height
+            notifications.db_height_changed = db.db_flushed_event.wait
+            notifications.cached_height = daemon.cached_height
+            notifications.daemon_height_changed = daemon.height_changed.wait
+            notifications.mempool_txids_hum = daemon.mempool_txids_hum
+            notifications.raw_transactions = daemon.getrawtransactions
+            notifications.lookup_utxos = db.lookup_utxos
+            MemPoolAPI.register(Notifications)
+            mempool = MemPool(
+                env.coin, notifications,
+                refresh_secs=env.daemon_poll_interval_mempool_msec/1000,
+            )
+
+            session_mgr = SessionManager(
+                env=env,
+                db=db,
+                block_processor=bp,
+                daemon=daemon,
+                mempool=mempool,
+                shutdown_event=shutdown_event,
+            )
+
+            # Test daemon authentication, and also ensure it has a cached
+            # height. Do this before entering the task group.
+            await daemon.height()
+
+            # Check if daemon is recent enough
+            # note: we only check the first daemon
+            #       (DAEMON_URL might be set to a list of daemons)
+            await daemon.check_daemon_version()
+            await daemon.check_daemon_indexes()
+
+            caught_up_event = Event()
+            mempool_event = Event()
+
+            async def wait_for_catchup():
+                await caught_up_event.wait()
+                await group.spawn(db.populate_header_merkle_cache())
+                await group.spawn(mempool.keep_synchronized(mempool_event))
+
+            try:
+                async with OldTaskGroup() as group:
+                    await group.spawn(session_mgr.serve(notifications, mempool_event))
+                    await group.spawn(bp.fetch_and_process_blocks(caught_up_event))
+                    await group.spawn(wait_for_catchup())
+            except Exception as e:
+                self.logger.exception("taskgroup died.")
+            finally:
+                self.logger.info("taskgroup stopped.")

src/electrumx/server/daemon.py

@@ -0,0 +1,638 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Class for handling asynchronous connections to a blockchain
+daemon.'''
+
+import asyncio
+import itertools
+import time
+from calendar import timegm
+from struct import pack
+from typing import TYPE_CHECKING, Type, Sequence, Any, Iterable, Optional
+
+import aiohttp
+from aiorpcx import JSONRPC
+
+from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash
+from electrumx.lib.tx import DeserializerDecred
+from electrumx.lib.util import (class_logger, hex_to_bytes, json_deserialize,
+                                json_serialize, pack_varint,
+                                unpack_le_uint16_from)
+
+if TYPE_CHECKING:
+    from electrumx.lib.coins import Coin
+
+
+class DaemonError(Exception):
+    '''Raised when the daemon returns an error in its results.'''
+
+
+class WarmingUpError(Exception):
+    '''Internal - when the daemon is warming up.'''
+
+
+class ServiceRefusedError(Exception):
+    '''Internal - when the daemon doesn't provide a JSON response, only an HTTP error, for
+    some reason.'''
+
+
+class DaemonParseError(Exception):
+    '''Internal - when the daemon returns an RPC_PARSE_ERROR.'''
+
+
+class Daemon:
+    '''Handles connections to a daemon at the given URL.'''
+
+    RPC_IN_WARMUP = -28
+    RPC_PARSE_ERROR = -32700
+
+    id_counter = itertools.count()
+
+    def __init__(
+            self,
+            coin: Type['Coin'],
+            url,
+            *,
+            max_workqueue=10,
+            init_retry=0.25,
+            max_retry=4.0,
+    ):
+        self.coin = coin
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.url_index = None
+        self.urls = []
+        self.set_url(url)
+        # Limit concurrent RPC calls to this number.
+        # See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16
+        self.workqueue_semaphore = asyncio.Semaphore(value=max_workqueue)
+        self.init_retry = init_retry
+        self.max_retry = max_retry
+        self._height = None
+        self.height_changed = asyncio.Event()
+        self.available_rpcs = {}
+        self.session = None
+
+        self._networkinfo_cache = (None, 0)
+        self._networkinfo_lock = asyncio.Lock()
+
+        self._mempoolinfo_cache = (None, 0)
+        self._mempoolinfo_lock = asyncio.Lock()
+
+    async def __aenter__(self):
+        self.session = aiohttp.ClientSession(connector=self.connector())
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.session.close()
+        self.session = None
+
+    def connector(self):
+        return None
+
+    async def check_daemon_version(self):
+        assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}"
+        if self.coin.MIN_REQUIRED_DAEMON_VERSION is None:
+            return
+
+        required_version = tuple(map(int, self.coin.MIN_REQUIRED_DAEMON_VERSION.split('.')))
+        network_info = await self.getnetworkinfo()
+        ni_version = network_info['version']
+        daemon_major = ni_version // 10_000
+        daemon_minor = (ni_version % 10_000) // 100
+        daemon_rev = ni_version % 100
+        daemon_version = (daemon_major, daemon_minor, daemon_rev)
+        if daemon_version < required_version:
+            raise RuntimeError(f"Bitcoin Core {daemon_version=} < {required_version=}.")
+
+    async def check_daemon_indexes(self):
+        assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}"
+        if not self.coin.REQUIRED_DAEMON_INDEXES:
+            return
+        index_info = await self.getindexinfo()
+        for req_index in self.coin.REQUIRED_DAEMON_INDEXES:
+            if req_index not in index_info:
+                raise RuntimeError(
+                    f"bitcoind missing required index: {req_index}. "
+                    f"You should set {req_index}=1 in your bitcoin.conf config file, and reindex.")
+            if not index_info[req_index]["synced"]:
+                # Should we raise? Not raising allows syncing a fresh bitcoind and e-x "in parallel".
+                self.logger.warning(f"bitcoind required index {req_index!r} is still syncing!")
+
+    def set_url(self, url: str) -> None:
+        '''Set the URLS to the given list, and switch to the first one.'''
+        urls = url.split(',')
+        urls = [self.coin.sanitize_url(url) for url in urls]
+        for n, url in enumerate(urls):
+            status = '' if n else ' (current)'
+            logged_url = self.logged_url(url)
+            self.logger.info(f'daemon #{n + 1} at {logged_url}{status}')
+        self.url_index = 0
+        self.urls = urls
+
+    def current_url(self) -> str:
+        '''Returns the current daemon URL.'''
+        return self.urls[self.url_index]
+
+    def logged_url(self, url=None):
+        '''The host and port part, for logging.'''
+        url = url or self.current_url()
+        return url[url.rindex('@') + 1:]
+
+    def failover(self):
+        '''Call to fail-over to the next daemon URL.
+
+        Returns False if there is only one, otherwise True.
+        '''
+        if len(self.urls) > 1:
+            self.url_index = (self.url_index + 1) % len(self.urls)
+            self.logger.info(f'failing over to {self.logged_url()}')
+            return True
+        return False
+
+    async def _send_data(self, data):
+        async with self.workqueue_semaphore:
+            async with self.session.post(self.current_url(), data=data) as resp:
+                kind = resp.headers.get('Content-Type', None)
+                if kind == 'application/json':
+                    return await resp.json(loads=json_deserialize)
+                text = await resp.text()
+                text = text.strip() or resp.reason
+                raise ServiceRefusedError(text)
+
+    async def _send(self, payload, processor):
+        '''Send a payload to be converted to JSON.
+
+        Handles temporary connection issues. Daemon response errors
+        are raise through DaemonError.
+        '''
+        def log_error(error):
+            nonlocal last_error_log, retry
+            now = time.monotonic()
+            if now - last_error_log > 60:
+                last_error_log = now
+                self.logger.error(f'{error}.  Retrying occasionally...')
+            if retry == self.max_retry and self.failover():
+                retry = 0
+
+        on_good_message = None
+        last_error_log = 0
+        data = json_serialize(payload)
+        retry = self.init_retry
+        while True:
+            try:
+                result = await self._send_data(data)
+                result = processor(result)
+                if on_good_message:
+                    self.logger.info(on_good_message)
+                return result
+            except asyncio.TimeoutError:
+                log_error('timeout error')
+            except aiohttp.ServerDisconnectedError:
+                log_error('disconnected')
+                on_good_message = 'connection restored'
+            except ConnectionResetError:
+                log_error('connection reset')
+                on_good_message = 'connection restored'
+            except aiohttp.ClientConnectionError:
+                log_error('connection problem - check your daemon is running')
+                on_good_message = 'connection restored'
+            except aiohttp.ClientError as e:
+                log_error(f'daemon error: {e}')
+                on_good_message = 'running normally'
+            except ServiceRefusedError as e:
+                log_error(f'daemon service refused: {e}')
+                on_good_message = 'running normally'
+            except WarmingUpError:
+                log_error('starting up checking blocks')
+                on_good_message = 'running normally'
+            except DaemonParseError as e:
+                log_error(f"received RPC_PARSE_ERROR ({e=!r}). request was: {data=!r}")
+                on_good_message = 'running normally'
+
+            await asyncio.sleep(retry)
+            retry = max(min(self.max_retry, retry * 2), self.init_retry)
+
+    async def _send_single(self, method, params=None):
+        '''Send a single request to the daemon.'''
+        def processor(result):
+            err = result['error']
+            if not err:
+                return result['result']
+            if err.get('code') == self.RPC_IN_WARMUP:
+                raise WarmingUpError
+            if err.get('code') == self.RPC_PARSE_ERROR:
+                raise DaemonParseError(err)
+            raise DaemonError(err)
+
+        payload = {'method': method, 'id': next(self.id_counter)}
+        if params:
+            payload['params'] = params
+        return await self._send(payload, processor)
+
+    async def _send_vector(self, method, params_iterable, replace_errs=False):
+        '''Send several requests of the same method.
+
+        The result will be an array of the same length as params_iterable.
+        If replace_errs is true, any item with an error is returned as None,
+        otherwise an exception is raised.'''
+        def processor(result):
+            errs = [item['error'] for item in result if item['error']]
+            if any(err.get('code') == self.RPC_IN_WARMUP for err in errs):
+                raise WarmingUpError
+            if any(err.get('code') == self.RPC_PARSE_ERROR for err in errs):
+                raise DaemonParseError
+            if not errs or replace_errs:
+                return [item['result'] for item in result]
+            raise DaemonError(errs)
+
+        payload = [{'method': method, 'params': p, 'id': next(self.id_counter)}
+                   for p in params_iterable]
+        if payload:
+            return await self._send(payload, processor)
+        return []
+
+    async def _is_rpc_available(self, method: str) -> bool:
+        '''Return whether given RPC method is available in the daemon.
+
+        Results are cached and the daemon will generally not be queried with
+        the same method more than once.'''
+        available = self.available_rpcs.get(method)
+        if available is None:
+            available = True
+            try:
+                await self._send_single(method)
+            except DaemonError as e:
+                err = e.args[0]
+                error_code = err.get("code")
+                available = error_code != JSONRPC.METHOD_NOT_FOUND
+            self.available_rpcs[method] = available
+        return available
+
+    async def block_hex_hashes(self, first: int, count: int) -> Sequence[str]:
+        '''Return the hex hashes (hum) of count block starting at height first.'''
+        params_iterable = ((h, ) for h in range(first, first + count))
+        return await self._send_vector('getblockhash', params_iterable)
+
+    async def deserialised_block(self, bhash_hum: str) -> dict:
+        '''Return the deserialised block with the given hex hash.'''
+        return await self._send_single('getblock', (bhash_hum, True))
+
+    async def raw_blocks(self, bhashes_hum: Sequence[str]) -> Sequence[bytes]:
+        '''Return the raw binary blocks with the given hex hashes.'''
+        params_iterable = ((h, False) for h in bhashes_hum)
+        blocks = await self._send_vector('getblock', params_iterable)
+        # Convert hex string to bytes
+        return [hex_to_bytes(block) for block in blocks]
+
+    async def mempool_txids_hum(self) -> Sequence[str]:
+        '''Update our record of the daemon's mempool hashes.'''
+        return await self._send_single('getrawmempool')
+
+    async def estimatefee(self, block_count, estimate_mode=None):
+        '''Return the fee estimate for the block count.  Units are whole
+        currency units per KB, e.g. 0.00000995, or -1 if no estimate
+        is available.
+        '''
+        if estimate_mode:
+            args = (block_count, estimate_mode)
+        else:
+            args = (block_count, )
+        if await self._is_rpc_available('estimatesmartfee'):
+            estimate = await self._send_single('estimatesmartfee', args)
+            return estimate.get('feerate', -1)
+        return await self._send_single('estimatefee', args)
+
+    async def getnetworkinfo(self):
+        '''Return the result of the 'getnetworkinfo' RPC call.'''
+        async with self._networkinfo_lock:
+            cache_val, cache_time = self._networkinfo_cache
+            if time.time() - cache_time < 60:  # seconds
+                return cache_val
+            val = await self._send_single('getnetworkinfo')
+            self._networkinfo_cache = (val, time.time())
+            return val
+
+    async def getmempoolinfo(self):
+        """Return the result of the 'getmempoolinfo' RPC call."""
+        async with self._mempoolinfo_lock:
+            cache_val, cache_time = self._mempoolinfo_cache
+            if time.time() - cache_time < 60:  # seconds
+                return cache_val
+            val = await self._send_single('getmempoolinfo')
+            self._mempoolinfo_cache = (val, time.time())
+            return val
+
+    async def getindexinfo(self):
+        """Return the result of the 'getindexinfo' RPC call."""
+        # note: not cached as it's not exposed to client sessions
+        return await self._send_single('getindexinfo')
+
+    async def relayfee(self):
+        """Same as getmempoolinfo['minrelaytxfee'].
+        The minimum fee required for a transaction to be relayed on by the daemon to the
+        bitcoin network. Doesn't guarantee mempool acceptance."""
+        network_info = await self.getnetworkinfo()
+        return network_info['relayfee']
+
+    async def mempool_info(self) -> dict[str, float]:
+        mempool_info = await self.getmempoolinfo()
+        return {
+            # Dynamic minimum fee rate in BTC/kvB for tx to be accepted.
+            # Is the maximum of minrelaytxfee and minimum mempool fee
+            'mempoolminfee': mempool_info['mempoolminfee'],
+            # Minimum relay fee for transactions in BTC/kvB, operator-configurable, static.
+            'minrelaytxfee': mempool_info['minrelaytxfee'],
+            # Minimum fee rate increment for mempool limiting or replacement in BTC/kvB,
+            # operator-configurable, static.
+            'incrementalrelayfee': mempool_info['incrementalrelayfee'],
+        }
+
+    async def getrawtransaction(self, txid_hum: str, verbose=False):
+        '''Return the serialized raw transaction with the given hash.'''
+        # Cast to int because some coin daemons are old and require it
+        return await self._send_single('getrawtransaction',
+                                       (txid_hum, int(verbose)))
+
+    async def getrawtransactions(self, txids_hum: Iterable[str], replace_errs=True) -> Sequence[bytes | None]:
+        '''Return the serialized raw transactions with the given hashes.
+
+        Replaces errors with None by default.'''
+        params_iterable = ((txid_hum, 0) for txid_hum in txids_hum)
+        txs = await self._send_vector('getrawtransaction', params_iterable,
+                                      replace_errs=replace_errs)
+        # Convert hex strings to bytes
+        return [hex_to_bytes(tx) if tx else None for tx in txs]
+
+    async def gettxspendingprevout(self, prev_txid_hum: str, txout_idx: int) -> dict[str, Any]:
+        """Query the daemon to find (if any) the spender of given outpoint."""
+        outpoints = [{"txid": prev_txid_hum, "vout": txout_idx}, ]
+        options = {"mempool_only": False}
+        tx_items = await self._send_single('gettxspendingprevout', (outpoints, options))
+        return tx_items[0]
+
+    async def broadcast_transaction(self, raw_tx: str) -> str:
+        '''Broadcast a transaction to the network.'''
+        return await self._send_single('sendrawtransaction', (raw_tx, ))
+
+    async def broadcast_package(self, raw_txs: Sequence[str]):
+        """Broadcast a package of transactions to the network using 'submitpackage'."""
+        return await self._send_single('submitpackage', (raw_txs, ))
+
+    async def testmempoolaccept(self, raw_txs: Sequence[str]):
+        """Query the daemon to test mempool acceptance of txs,
+        without adding them to the mempool or broadcasting them.
+        """
+        return await self._send_single('testmempoolaccept', (raw_txs, ))
+
+    async def height(self) -> int:
+        '''Query the daemon for its current height.'''
+        old_height = self._height
+        self._height = await self._send_single('getblockcount')
+        if old_height != self._height:
+            self.logger.debug(f"new height: {self._height}")
+            self.height_changed.set()
+            self.height_changed.clear()
+        return self._height
+
+    def cached_height(self) -> Optional[int]:
+        '''Return the cached daemon height.
+
+        If the daemon has not been queried yet this returns None.'''
+        return self._height
+
+
+class DashDaemon(Daemon):
+
+    async def masternode_broadcast(self, params):
+        '''Broadcast a transaction to the network.'''
+        return await self._send_single('masternodebroadcast', params)
+
+    async def masternode_list(self, params):
+        '''Return the masternode status.'''
+        return await self._send_single('masternodelist', params)
+
+    async def protx(self, params):
+        '''Set of commands to execute ProTx related actions.'''
+        return await self._send_single('protx', params)
+
+
+class FakeEstimateFeeDaemon(Daemon):
+    '''Daemon that simulates estimatefee and relayfee RPC calls. Coin that
+    wants to use this daemon must define ESTIMATE_FEE & RELAY_FEE'''
+
+    async def estimatefee(self, block_count):
+        '''Return the fee estimate for the given parameters.'''
+        return self.coin.ESTIMATE_FEE
+
+    async def relayfee(self):
+        """The minimum fee required for a transaction to be relayed on by the server to the
+        bitcoin network. Doesn't guarantee mempool acceptance."""
+        return self.coin.RELAY_FEE
+
+
+class LegacyRPCDaemon(Daemon):
+    '''Handles connections to a daemon at the given URL.
+
+    This class is useful for daemons that don't have the new 'getblock'
+    RPC call that returns the block in hex, the workaround is to manually
+    recreate the block bytes. The recreated block bytes may not be the exact
+    as in the underlying blockchain but it is good enough for our indexing
+    purposes.'''
+
+    async def raw_blocks(self, bhashes_hum):
+        '''Return the raw binary blocks with the given hex hashes.'''
+        params_iterable = ((h, ) for h in bhashes_hum)
+        block_info = await self._send_vector('getblock', params_iterable)
+
+        blocks = []
+        for i in block_info:
+            raw_block = await self.make_raw_block(i)
+            blocks.append(raw_block)
+
+        # Convert hex string to bytes
+        return blocks
+
+    async def make_raw_header(self, b):
+        pbh = b.get('previousblockhash')
+        if pbh is None:
+            pbh = '0' * 64
+        return b''.join([
+            pack('<L', b.get('version')),
+            hex_str_to_hash(pbh),
+            hex_str_to_hash(b.get('merkleroot')),
+            pack('<L', self.timestamp_safe(b['time'])),
+            pack('<L', int(b.get('bits'), 16)),
+            pack('<L', int(b.get('nonce')))
+        ])
+
+    async def make_raw_block(self, b):
+        '''Construct a raw block'''
+
+        header = await self.make_raw_header(b)
+
+        transactions = []
+        if b.get('height') > 0:
+            transactions = await self.getrawtransactions(b.get('tx'), False)
+
+        raw_block = header
+        num_txs = len(transactions)
+        if num_txs > 0:
+            raw_block += pack_varint(num_txs)
+            raw_block += b''.join(transactions)
+        else:
+            raw_block += b'\x00'
+
+        return raw_block
+
+    def timestamp_safe(self, t):
+        if isinstance(t, int):
+            return t
+        return timegm(time.strptime(t, "%Y-%m-%d %H:%M:%S %Z"))
+
+
+class FakeEstimateLegacyRPCDaemon(LegacyRPCDaemon, FakeEstimateFeeDaemon):
+    pass
+
+
+class DecredDaemon(Daemon):
+    async def raw_blocks(self, bhashes_hum):
+        '''Return the raw binary blocks with the given hex hashes.'''
+
+        params_iterable = ((h, False) for h in bhashes_hum)
+        blocks = await self._send_vector('getblock', params_iterable)
+
+        raw_blocks = []
+        valid_tx_tree = {}
+        for block in blocks:
+            # Convert to bytes from hex
+            raw_block = hex_to_bytes(block)
+            raw_blocks.append(raw_block)
+            # Check if previous block is valid
+            prev = self.prev_hex_hash(raw_block)
+            votebits = unpack_le_uint16_from(raw_block[100:102])[0]
+            valid_tx_tree[prev] = self.is_valid_tx_tree(votebits)
+
+        processed_raw_blocks = []
+        for hash, raw_block in zip(bhashes_hum, raw_blocks):
+            if hash in valid_tx_tree:
+                is_valid = valid_tx_tree[hash]
+            else:
+                # Do something complicated to figure out if this block is valid
+                header = await self._send_single('getblockheader', (hash, ))
+                if 'nextblockhash' not in header:
+                    raise DaemonError(f'Could not find next block for {hash}')
+                next_hash = header['nextblockhash']
+                next_header = await self._send_single('getblockheader',
+                                                      (next_hash, ))
+                is_valid = self.is_valid_tx_tree(next_header['votebits'])
+
+            if is_valid:
+                processed_raw_blocks.append(raw_block)
+            else:
+                # If this block is invalid remove the normal transactions
+                self.logger.info(f'block {hash} is invalidated')
+                processed_raw_blocks.append(self.strip_tx_tree(raw_block))
+
+        return processed_raw_blocks
+
+    @staticmethod
+    def prev_hex_hash(raw_block):
+        return hash_to_hex_str(raw_block[4:36])
+
+    @staticmethod
+    def is_valid_tx_tree(votebits):
+        # Check if previous block was invalidated.
+        return bool(votebits & (1 << 0) != 0)
+
+    def strip_tx_tree(self, raw_block):
+        c = self.coin
+        assert issubclass(c.DESERIALIZER, DeserializerDecred)
+        d = c.DESERIALIZER(raw_block, start=c.BASIC_HEADER_SIZE)
+        d.read_tx_tree()  # Skip normal transactions
+        # Create a fake block without any normal transactions
+        return raw_block[:c.BASIC_HEADER_SIZE] + b'\x00' + raw_block[d.cursor:]
+
+    async def height(self):
+        height = await super().height()
+        if height > 0:
+            # Lie about the daemon height as the current tip can be invalidated
+            height -= 1
+            self._height = height
+        return height
+
+    async def mempool_txids_hum(self):
+        mempool = await super().mempool_txids_hum()
+        # Add current tip transactions to the 'fake' mempool.
+        real_height = await self._send_single('getblockcount')
+        tip_hash = await self._send_single('getblockhash', (real_height,))
+        tip = await self.deserialised_block(tip_hash)
+        # Add normal transactions except coinbase
+        mempool += tip['tx'][1:]
+        # Add stake transactions if applicable
+        mempool += tip.get('stx', [])
+        return mempool
+
+    def connector(self):
+        # FIXME allow self signed certificates
+        return aiohttp.TCPConnector(verify_ssl=False)
+
+
+class PreLegacyRPCDaemon(LegacyRPCDaemon):
+    '''Handles connections to a daemon at the given URL.
+
+    This class is useful for daemons that don't have the new 'getblock'
+    RPC call that returns the block in hex, and need the False parameter
+    for the getblock'''
+
+    async def deserialised_block(self, hex_hash):
+        '''Return the deserialised block with the given hex hash.'''
+        return await self._send_single('getblock', (hex_hash, False))
+
+
+class SmartCashDaemon(Daemon):
+
+    async def masternode_broadcast(self, params):
+        '''Broadcast a smartnode to the network.'''
+        return await self._send_single('smartnodebroadcast', params)
+
+    async def masternode_list(self, params):
+        '''Return the smartnode status.'''
+        return await self._send_single('smartnodelist', params)
+
+    async def smartrewards(self, params):
+        '''Return smartrewards data.'''
+        return await self._send_single('smartrewards', params)
+
+
+class ZcoinMtpDaemon(Daemon):
+
+    def strip_mtp_data(self, raw_block):
+        if self.coin.is_mtp(raw_block):
+            return \
+                raw_block[:self.coin.MTP_HEADER_DATA_START*2] + \
+                raw_block[self.coin.MTP_HEADER_DATA_END*2:]
+        return raw_block
+
+    async def raw_blocks(self, bhashes_hum):
+        '''Return the raw binary blocks with the given hex hashes.'''
+        params_iterable = ((h, False) for h in bhashes_hum)
+        blocks = await self._send_vector('getblock', params_iterable)
+        # Convert hex string to bytes
+        return [hex_to_bytes(self.strip_mtp_data(block)) for block in blocks]
+
+    async def masternode_broadcast(self, params):
+        '''Broadcast a transaction to the network.'''
+        return await self._send_single('znodebroadcast', params)
+
+    async def masternode_list(self, params):
+        '''Return the masternode status.'''
+        return await self._send_single('znodelist', params)
+
+    async def protx(self, params):
+        '''Set of commands to execute ProTx related actions.'''
+        return await self._send_single('protx', params)

src/electrumx/server/db.py

@@ -0,0 +1,820 @@
+# Copyright (c) 2016-2020, Neil Booth
+# Copyright (c) 2017, the ElectrumX authors
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Interface to the blockchain database.'''
+
+
+from array import array
+import ast
+import asyncio
+import os
+import time
+from bisect import bisect_right
+from dataclasses import dataclass
+from glob import glob
+from typing import Dict, List, Sequence, Tuple, Optional, TYPE_CHECKING, Union, Callable, Iterable
+
+from aiorpcx import run_in_thread, sleep
+
+import electrumx.lib.util as util
+from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN, hex_str_to_hash
+from electrumx.lib.merkle import Merkle, MerkleCache
+from electrumx.lib.util import (
+    formatted_time,
+)
+from electrumx.lib.tx import TXOSpendStatus
+from electrumx.server.storage import db_class, Storage
+from electrumx.server.history import (
+    History,
+)
+from electrumx.server.db_util import (
+    DBTooOldForMigrations,
+    pack_txnum, unpack_txnum, TXNUM_LEN,
+    pack_txoutidx, unpack_txoutidx, TXOUTIDX_LEN,
+    pack_satoshis_val, unpack_satoshis_val,
+    pack_block_height, unpack_block_height, BHEIGHT_LEN,
+    pack_dyn_header_offset, unpack_dyn_header_offset, DYN_HEADER_OFFSET_LEN,
+)
+
+if TYPE_CHECKING:
+    from electrumx.server.env import Env
+
+
+@dataclass(slots=True, order=True)
+class UTXO:
+    tx_num: int      # index of tx in chain order
+    tx_pos: int      # tx output idx
+    txid_rev: bytes | None
+    height: int      # block height
+    value: int       # in satoshis
+
+
+@dataclass(slots=True, kw_only=True)
+class FlushData:
+    height: int
+    tx_count: int
+    headers: list[bytes]
+    bhash_to_bheight: dict[bytes, int]  # block_hash_rev -> block_height
+    block_txids_rev: list[bytes]
+    # The following are flushed to the UTXO DB if undo_infos is not None
+    undo_infos: dict[int, Sequence[bytes]]  # height -> undo_info
+    adds: dict[bytes, bytes]  # txid+out_idx -> hashX+tx_num+value_sats
+    deletes: list[bytes]  # b'h' db keys, and b'u' db keys.  order does not matter
+    tip: bytes
+
+
+COMP_TXID_LEN = 4
+
+
+class DB:
+    '''Simple wrapper of the backend database for querying.
+
+    Performs no DB update, though the DB will be cleaned on opening if
+    it was shutdown uncleanly.
+    '''
+
+    DB_VERSIONS = (11, )
+    DB_STATE_KEY = b'state'
+
+    utxo_db: Optional['Storage']
+
+    class DBError(Exception):
+        '''Raised on general DB errors generally indicating corruption.'''
+
+    def __init__(self, env: 'Env'):
+        self.logger = util.class_logger(__name__, self.__class__.__name__)
+        self.env = env
+        self.coin = env.coin
+
+        # Setup block header size handlers
+        if self.coin.STATIC_BLOCK_HEADERS:
+            self.header_offset = self.coin.static_header_offset  # type: Callable[[int], int]
+            self.header_len = self.coin.static_header_len  # type: Callable[[int], int]
+        else:
+            self.header_offset = self.dynamic_header_offset  # type: Callable[[int], int]
+            self.header_len = self.dynamic_header_len  # type: Callable[[int], int]
+
+        self.logger.info(f'switching current directory to {env.db_dir}')
+        os.chdir(env.db_dir)
+
+        self.db_class = db_class(self.env.db_engine)
+        self.history = History()
+
+        # Key: b'u' + address_hashX + txout_idx + tx_num
+        # Value: the UTXO value as a 64-bit unsigned integer (in satoshis)
+        # "at address, at outpoint, there is a UTXO of value v"
+        # ---
+        # Key: b'h' + compressed_tx_hash + txout_idx + tx_num
+        # Value: hashX
+        # "some outpoint created a UTXO at address"
+        # ---
+        # Key: b'U' + block_height
+        # Value: byte-concat list of (hashX + tx_num + value_sats)
+        # "undo data: list of UTXOs spent at block height"
+        self.utxo_db = None
+
+        self.fs_height = -1
+        self.fs_tx_count = 0
+        self.db_height = -1
+        self.db_tx_count = 0
+        self.db_tip = None  # type: Optional[bytes]  # block_hash_rev
+        self.last_flush = time.time()
+        self.last_flush_tx_count = 0
+        self.wall_time = 0
+        self.first_sync = True
+        self.db_version = -1
+        self.db_flushed_event = asyncio.Event()
+
+        self.logger.info(f'using {self.env.db_engine} for DB backend')
+
+        # Header merkle cache
+        self.merkle = Merkle()
+        self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes_rev)
+
+        # note: all on-disk and in-memory dbs are consistent with each other if BlockProcessor.tip == DB.tip.
+        #       (the mempool is out-of-scope here)
+
+        # on-disk: raw block headers in chain order
+        self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
+        # on-disk: cumulative number of txs at the end of height N
+        self.tx_counts = None  # type: Optional[array]  # in-memory
+        self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
+        # on-disk: 32 byte txids in chain order, allows (tx_num -> txid_rev) map
+        self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
+        if not self.coin.STATIC_BLOCK_HEADERS:
+            self.headers_offsets_file = util.LogicalFile(
+                'meta/headers_offsets', 2, 16000000)
+
+        # in-memory: (block_hash_rev -> block_height) full map
+        self.bhash_to_bheight = None  # type: Optional[dict[bytes, int]]
+
+    async def _read_tx_counts(self) -> None:
+        if self.tx_counts is not None:
+            return
+        # tx_counts[N] has the cumulative number of txs at the end of
+        # height N.  So tx_counts[0] is 1 - the genesis coinbase
+        size = (self.db_height + 1) * 8
+        tx_counts = self.tx_counts_file.read(0, size)
+        assert len(tx_counts) == size
+        self.tx_counts = array('Q', tx_counts)
+        if self.tx_counts:
+            assert self.db_tx_count == self.tx_counts[-1]
+        else:
+            assert self.db_tx_count == 0
+
+    async def _open_dbs(self, *, for_sync: bool) -> None:
+        assert self.utxo_db is None
+
+        # First UTXO DB
+        def open_utxo_db():
+            self.utxo_db = self.db_class('utxo', for_sync)
+            if self.utxo_db.is_new:
+                self.logger.info('created new database')
+                self.logger.info('creating metadata directory')
+                os.mkdir('meta')
+                with util.open_file('COIN', create=True) as f:
+                    f.write(f'ElectrumX databases and metadata for '
+                            f'{self.coin.NAME} {self.coin.NET}'.encode())
+                if not self.coin.STATIC_BLOCK_HEADERS:
+                    self.headers_offsets_file.write(0, b'\0\0\0\0\0\0\0\0')
+            else:
+                self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
+            self.read_utxo_state()
+        await run_in_thread(open_utxo_db)
+
+        # Then history DB
+        def open_hist_db():
+            self.history.open_db(
+                db_class=self.db_class,
+                for_sync=for_sync,
+                utxo_db_tx_count=self.db_tx_count,
+            )
+            self.clear_excess_undo_info()
+        await run_in_thread(open_hist_db)
+
+        # Now prepare in-memory structures.
+        # - Read TX counts (requires meta directory)
+        await self._read_tx_counts()
+        # - (block_hash_rev -> block_height) map
+        await self._prep_bhash_to_bheight_map()
+
+    async def open_for_sync(self) -> None:
+        '''Open the databases to sync to the daemon.
+
+        When syncing we want to reserve a lot of open files for the
+        synchronization.  When serving clients we want the open files for
+        serving network connections.
+        '''
+        await self._open_dbs(for_sync=True)
+
+    async def open_for_serving(self) -> None:
+        '''Open the databases for serving.  If they are already open they are
+        closed first.
+        '''
+        if self.utxo_db:
+            self.logger.info('closing DBs to re-open for serving')
+            self.utxo_db.close()
+            self.history.close_db()
+            self.utxo_db = None
+        await self._open_dbs(for_sync=False)
+
+    # Header merkle cache
+
+    async def populate_header_merkle_cache(self) -> None:
+        self.logger.info('populating header merkle cache...')
+        length = max(1, self.db_height - self.env.reorg_limit)
+        start = time.monotonic()
+        await self.header_mc.initialize(length)
+        elapsed = time.monotonic() - start
+        self.logger.info(f'header merkle cache populated in {elapsed:.1f}s')
+
+    async def header_branch_and_root(self, length: int, height: int) -> tuple[list[bytes], bytes]:
+        return await self.header_mc.branch_and_root(length, height)
+
+    # -- (block_hash -> block_height) map
+    async def _prep_bhash_to_bheight_map(self) -> None:
+        if self.bhash_to_bheight is not None:
+            return
+        self.logger.info('preparing bhash->bheight map...')
+        t0 = time.monotonic()
+        self.bhash_to_bheight = {}
+        count = self.db_height + 1
+        block_hashes = await self.fs_block_hashes_rev(0, count)
+        if len(block_hashes) != count:
+            raise Exception(
+                f"failed to prep bhash_to_bheight. "
+                f"wanted {count} bhashes, only got {len(block_hashes)}")
+        for bheight, bhash in enumerate(block_hashes):
+            self.bhash_to_bheight[bhash] = bheight
+        assert len(self.bhash_to_bheight) == self.db_height + 1
+        self.logger.info(f'bhash->bheight map populated in {time.monotonic() - t0:.1f}s')
+
+    def get_blockheight_from_blockhash(self, block_hash_hum: str) -> Optional[int]:
+        bhash_rev = hex_str_to_hash(block_hash_hum)
+        return self.bhash_to_bheight.get(bhash_rev, None)
+
+    # Flushing
+    def assert_flushed(self, flush_data: FlushData) -> None:
+        '''Asserts state is fully flushed.'''
+        assert flush_data.tx_count == self.fs_tx_count == self.db_tx_count
+        assert flush_data.height == self.fs_height == self.db_height
+        assert flush_data.tip == self.db_tip
+        assert not flush_data.headers
+        assert not flush_data.bhash_to_bheight
+        assert not flush_data.block_txids_rev
+        assert not flush_data.adds
+        assert not flush_data.deletes
+        assert not flush_data.undo_infos
+        self.history.assert_flushed()
+
+    def flush_dbs(
+            self,
+            *,
+            flush_data: FlushData,
+            flush_utxos: bool,
+            estimate_txs_remaining: Callable[[], float],
+    ) -> None:
+        '''Flush out cached state.  History is always flushed; UTXOs are
+        flushed if flush_utxos.'''
+        if flush_data.height == self.db_height:
+            assert flush_data.tip == self.db_tip, f"block_hash mismatch at {flush_data.height}"
+            self.assert_flushed(flush_data)
+            return
+
+        start_time = time.time()
+        prior_flush = self.last_flush
+        tx_delta = flush_data.tx_count - self.last_flush_tx_count
+
+        # 1. Flush to file system
+        self._flush_fs(flush_data)
+
+        # 2. Then hist_db
+        self._flush_history()
+
+        # 3. Then utxo_db
+        # Flush state last as it reads the wall time.
+        with self.utxo_db.write_batch() as batch:
+            if flush_utxos:
+                self._flush_utxo_db(batch, flush_data)
+            self._flush_state(batch)
+
+        # Update and put the wall time again - otherwise we drop the
+        # time it took to commit the batch
+        self._flush_state(self.utxo_db)
+
+        elapsed = self.last_flush - start_time
+        self.logger.info(f'flush took '
+                         f'{elapsed:.1f}s.  Height {flush_data.height:,d} '
+                         f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
+
+        # Catch-up stats
+        if self.utxo_db.for_sync:
+            flush_interval = self.last_flush - prior_flush
+            tx_per_sec_gen = int(flush_data.tx_count / self.wall_time)
+            tx_per_sec_last = 1 + int(tx_delta / flush_interval)
+            eta = estimate_txs_remaining() / tx_per_sec_last
+            self.logger.info(f'tx/sec since genesis: {tx_per_sec_gen:,d}, '
+                             f'since last flush: {tx_per_sec_last:,d}')
+            self.logger.info(f'sync time: {formatted_time(self.wall_time)}  '
+                             f'ETA: {formatted_time(eta)}')
+
+    def _flush_fs(self, flush_data: FlushData) -> None:
+        '''Write headers, tx counts and block tx hashes to the filesystem.
+
+        The first height to write is self.fs_height + 1.  The FS
+        metadata is all append-only, so in a crash we just pick up
+        again from the height stored in the DB.
+        '''
+        prior_tx_count = (self.tx_counts[self.fs_height]
+                          if self.fs_height >= 0 else 0)
+        assert len(flush_data.block_txids_rev) == len(flush_data.headers)
+        assert len(flush_data.bhash_to_bheight) == len(flush_data.headers)
+        assert flush_data.height == self.fs_height + len(flush_data.headers)
+        assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
+                                       else 0)
+        assert len(self.tx_counts) == flush_data.height + 1
+        txids_rev = b''.join(flush_data.block_txids_rev)
+        flush_data.block_txids_rev.clear()
+        assert len(txids_rev) % 32 == 0
+        assert len(txids_rev) // 32 == flush_data.tx_count - prior_tx_count
+
+        # Write the headers, tx counts, and tx hashes
+        start_time = time.monotonic()
+        height_start = self.fs_height + 1
+        offset = self.header_offset(height_start)
+        self.headers_file.write(offset, b''.join(flush_data.headers))
+        self.fs_update_header_offsets(offset_start=offset, height_start=height_start, headers=flush_data.headers)
+        flush_data.headers.clear()
+        self.bhash_to_bheight.update(flush_data.bhash_to_bheight)
+        assert len(self.bhash_to_bheight) == flush_data.height + 1
+        flush_data.bhash_to_bheight.clear()
+
+        offset = height_start * self.tx_counts.itemsize
+        self.tx_counts_file.write(offset,
+                                  self.tx_counts[height_start:].tobytes())
+        offset = prior_tx_count * 32
+        self.hashes_file.write(offset, txids_rev)
+
+        self.fs_height = flush_data.height
+        self.fs_tx_count = flush_data.tx_count
+
+        if self.utxo_db.for_sync:
+            elapsed = time.monotonic() - start_time
+            self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
+
+    def _flush_history(self) -> None:
+        self.history.flush()
+
+    def _flush_utxo_db(self, batch, flush_data: FlushData) -> None:
+        '''Flush the cached DB writes and UTXO set to the batch.'''
+        # Care is needed because the writes generated by flushing the
+        # UTXO state may have keys in common with our write cache or
+        # may be in the DB already.
+        start_time = time.monotonic()
+        add_count = len(flush_data.adds)
+        spend_count = len(flush_data.deletes) // 2
+
+        # Spends
+        batch_delete = batch.delete
+        for key in sorted(flush_data.deletes):
+            batch_delete(key)
+        flush_data.deletes.clear()
+
+        # New UTXOs
+        batch_put = batch.put
+        for key, value in flush_data.adds.items():
+            # key: txid+out_idx, value: hashX+tx_num+value_sats
+            hashX = value[:HASHX_LEN]
+            txout_idx = key[-TXOUTIDX_LEN:]
+            tx_num = value[HASHX_LEN: HASHX_LEN+TXNUM_LEN]
+            value_sats = value[-8:]
+            suffix = txout_idx + tx_num
+            batch_put(b'h' + key[:COMP_TXID_LEN] + suffix, hashX)
+            batch_put(b'u' + hashX + suffix, value_sats)
+        flush_data.adds.clear()
+
+        # Add new undo information
+        self.flush_undo_infos(batch_put, flush_data.undo_infos)
+        flush_data.undo_infos.clear()
+        # Delete old undo information
+        if not self.utxo_db.for_sync:  # undo infos were only added if we are nearly caught up anyway
+            old_min_height = max(0, self.min_undo_height(self.db_height))
+            new_min_height = max(0, self.min_undo_height(flush_data.height))
+            for h in range(old_min_height, new_min_height):
+                key = self.undo_key(h)
+                batch_delete(key)
+
+        # Log stats
+        if self.utxo_db.for_sync:
+            block_count = flush_data.height - self.db_height
+            tx_count = flush_data.tx_count - self.db_tx_count
+            elapsed = time.monotonic() - start_time
+            self.logger.info(f'flushed {block_count:,d} blocks with '
+                             f'{tx_count:,d} txs, {add_count:,d} UTXO adds, '
+                             f'{spend_count:,d} spends in '
+                             f'{elapsed:.1f}s, committing...')
+
+        self.db_height = flush_data.height
+        self.db_tx_count = flush_data.tx_count
+        self.db_tip = flush_data.tip
+
+    def _flush_state(self, batch):
+        '''Flush chain state to the batch.'''
+        now = time.time()
+        self.wall_time += now - self.last_flush
+        self.last_flush = now
+        self.last_flush_tx_count = self.fs_tx_count
+        self.write_utxo_state(batch)
+
+    def flush_backup(self, flush_data: FlushData, touched_hashxs: Iterable[bytes]) -> None:
+        '''Like flush_dbs() but when backing up.  All UTXOs are flushed.'''
+        assert not flush_data.headers
+        assert not flush_data.block_txids_rev
+        assert flush_data.height < self.db_height
+        assert len(flush_data.bhash_to_bheight) == (self.db_height - flush_data.height)
+        self.history.assert_flushed()
+
+        start_time = time.time()
+        tx_delta = flush_data.tx_count - self.last_flush_tx_count
+
+        # 1. (fake-)Flush to file system: this just updates pointers
+        self._backup_fs(flush_data)
+
+        # Crucially, we flush utxo_db and hist_db in the reverse order compared to when advancing,
+        #   to maintain the invariant that hist_db_tx_count >= utxo_db_tx_count even if we crash.
+        # 2. Then utxo_db
+        with self.utxo_db.write_batch() as batch:
+            self._flush_utxo_db(batch, flush_data)
+            # Flush state last as it reads the wall time.
+            self._flush_state(batch)
+
+        # 3. Then hist_db
+        self.history.backup(
+            hashXs=touched_hashxs,
+            tx_count=flush_data.tx_count,
+        )
+
+        # Update and put the wall time again - otherwise we drop the
+        # time it took to commit the batch
+        self._flush_state(self.utxo_db)
+
+        elapsed = self.last_flush - start_time
+        self.logger.info(f'backup flush took '
+                         f'{elapsed:.1f}s.  Height {flush_data.height:,d} '
+                         f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
+
+    def fs_update_header_offsets(self, *, offset_start: int, height_start: int, headers: Sequence[bytes]) -> None:
+        if self.coin.STATIC_BLOCK_HEADERS:
+            return
+        offset = offset_start
+        offsets = []
+        for h in headers:
+            offset += len(h)
+            offsets.append(pack_dyn_header_offset(offset))
+        # For each header we get the offset of the next header, hence we
+        # start writing from the next height
+        pos = (height_start + 1) * DYN_HEADER_OFFSET_LEN
+        self.headers_offsets_file.write(pos, b''.join(offsets))
+
+    def dynamic_header_offset(self, height: int) -> int:
+        assert not self.coin.STATIC_BLOCK_HEADERS
+        offset = unpack_dyn_header_offset(self.headers_offsets_file.read(height * DYN_HEADER_OFFSET_LEN, DYN_HEADER_OFFSET_LEN))
+        return offset
+
+    def dynamic_header_len(self, height: int) -> int:
+        return self.dynamic_header_offset(height + 1)\
+               - self.dynamic_header_offset(height)
+
+    def _backup_fs(self, flush_data: FlushData) -> None:
+        '''Back up during a reorg.  This just updates our pointers.'''
+        self.fs_height = flush_data.height
+        self.fs_tx_count = flush_data.tx_count
+
+        for bhash, bheight in flush_data.bhash_to_bheight.items():
+            assert bheight == self.bhash_to_bheight.pop(bhash)
+        flush_data.bhash_to_bheight.clear()
+        assert len(self.bhash_to_bheight) == flush_data.height + 1
+
+        # Truncate header_mc: header count is 1 more than the height.
+        self.header_mc.truncate(flush_data.height + 1)
+
+    async def raw_header(self, height: int) -> bytes:
+        '''Return the binary header at the given height.'''
+        header, n = await self.read_headers(height, 1)
+        if n != 1:
+            raise IndexError(f'height {height:,d} out of range')
+        return header
+
+    async def read_headers(self, start_height: int, count: int) -> Tuple[bytes, int]:
+        '''Requires start_height >= 0, count >= 0.  Reads as many headers as
+        are available starting at start_height up to count.  This
+        would be zero if start_height is beyond self.db_height, for
+        example.
+
+        Returns a (binary, n) pair where binary is the concatenated
+        binary headers, and n is the count of headers returned.
+        '''
+        if start_height < 0 or count < 0:
+            raise self.DBError(f'{count:,d} headers starting at '
+                               f'{start_height:,d} not on disk')
+
+        def read_headers():
+            # Read some from disk
+            disk_count = max(0, min(count, self.db_height + 1 - start_height))
+            if disk_count:
+                offset = self.header_offset(start_height)
+                size = self.header_offset(start_height + disk_count) - offset
+                return self.headers_file.read(offset, size), disk_count
+            return b'', 0
+
+        return await run_in_thread(read_headers)
+
+    def fs_txid_rev(self, tx_num: int) -> Tuple[Optional[bytes], int]:
+        '''Return a pair (txid_rev, tx_height) for the given tx number.
+
+        If the tx_height is not on disk, returns (None, tx_height).'''
+        tx_height = bisect_right(self.tx_counts, tx_num)
+        if tx_height > self.db_height:
+            txid_rev = None
+        else:
+            txid_rev = self.hashes_file.read(tx_num * 32, 32)
+        return txid_rev, tx_height
+
+    def fs_txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]:
+        '''Return a list of txids_rev at given block height,
+        in the same order as in the block.
+        '''
+        if block_height > self.db_height:
+            raise self.DBError(f'block {block_height:,d} not on disk (>{self.db_height:,d})')
+        assert block_height >= 0
+        if block_height > 0:
+            first_tx_num = self.tx_counts[block_height - 1]
+        else:
+            first_tx_num = 0
+        num_txs_in_block = self.tx_counts[block_height] - first_tx_num
+        txids_rev = self.hashes_file.read(first_tx_num * 32, num_txs_in_block * 32)
+        assert num_txs_in_block == len(txids_rev) // 32
+        return [txids_rev[idx * 32: (idx+1) * 32] for idx in range(num_txs_in_block)]
+
+    async def txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]:
+        return await run_in_thread(self.fs_txids_rev_at_blockheight, block_height)
+
+    async def fs_block_hashes_rev(self, height: int, count: int) -> Sequence[bytes]:
+        headers_concat, headers_count = await self.read_headers(height, count)
+        if headers_count != count:
+            raise self.DBError(f'only got {headers_count:,d} headers starting '
+                               f'at {height:,d}, not {count:,d}')
+
+        def hash_headers():
+            offset = 0
+            headers = []
+            for n in range(count):
+                hlen = self.header_len(height + n)
+                headers.append(headers_concat[offset:offset + hlen])
+                offset += hlen
+            return [self.coin.header_hash_rev(header) for header in headers]
+
+        bhashes = (await run_in_thread(hash_headers)) if count > 100 else hash_headers()
+        return bhashes
+
+    async def limited_history(self, hashX: bytes, *, limit: int = 1000) -> Sequence[tuple[bytes, int]]:
+        '''Return an unpruned, sorted list of (txid_rev, height) tuples of
+        confirmed transactions that touched the address, earliest in
+        the blockchain first.  Includes both spending and receiving
+        transactions.  By default returns at most 1000 entries.  Set
+        limit to None to get them all.
+        '''
+        def read_history():
+            tx_nums = list(self.history.get_txnums(hashX, limit))
+            fs_txid_rev = self.fs_txid_rev
+            return [fs_txid_rev(tx_num) for tx_num in tx_nums]
+
+        while True:
+            history = await run_in_thread(read_history)
+            if all(txid_rev is not None for txid_rev, height in history):
+                return history
+            self.logger.warning(f'limited_history: tx hash '
+                                f'not found (reorg?), retrying...')
+            await sleep(0.25)
+
+    # -- Undo information
+
+    def min_undo_height(self, max_height: int) -> int:
+        '''Returns a height from which we should store undo info.'''
+        return max_height - self.env.reorg_limit + 1
+
+    def undo_key(self, height: int) -> bytes:
+        '''DB key for undo information at the given height.'''
+        return b'U' + pack_block_height(height)
+
+    def read_undo_info(self, height: int) -> bytes:
+        '''Read undo information from a file for the current height.'''
+        return self.utxo_db.get(self.undo_key(height))
+
+    def flush_undo_infos(
+            self, batch_put, undo_infos: dict[int, Sequence[bytes]],
+    ) -> None:
+        '''undo_infos is a (height -> undo_info) map.'''
+        for height, undo_info in undo_infos.items():
+            batch_put(self.undo_key(height), b''.join(undo_info))
+
+    def raw_block_prefix(self) -> str:
+        return 'meta/block'
+
+    def raw_block_path(self, height: int) -> str:
+        return f'{self.raw_block_prefix()}{height:d}'
+
+    def read_raw_block(self, height: int) -> bytes:
+        '''Returns a raw block read from disk.  Raises FileNotFoundError
+        if the block isn't on-disk.'''
+        with util.open_file(self.raw_block_path(height)) as f:
+            return f.read(-1)
+
+    def write_raw_block(self, block: bytes, height: int) -> None:
+        '''Write a raw block to disk.'''
+        with util.open_truncate(self.raw_block_path(height)) as f:
+            f.write(block)
+        # Delete old blocks to prevent them accumulating
+        del_height = self.min_undo_height(height) - 1
+        if del_height >= 0:
+            try:
+                os.remove(self.raw_block_path(del_height))
+            except FileNotFoundError:
+                pass
+
+    def clear_excess_undo_info(self) -> None:
+        '''Clear excess undo info.  Only most recent N are kept.'''
+        # delete aged undo_infos from utxo db
+        # note: this is just a fallback cleanup path - normally already deleted by _flush_utxo_db()
+        min_height = self.min_undo_height(self.db_height)
+        keys = []
+        for key, uhist in self.utxo_db.iterator(prefix=b'U'):
+            height = unpack_block_height(key[-BHEIGHT_LEN:])
+            if height >= min_height:
+                break  # can break as block_height is encoded as big endian
+            keys.append(key)
+
+        if keys:
+            with self.utxo_db.write_batch() as batch:
+                for key in keys:
+                    batch.delete(key)
+            self.logger.info(f'deleted {len(keys):,d} stale undo entries')
+
+        # delete old block files
+        # note: this is just a fallback cleanup path - normally already deleted by write_raw_block()
+        prefix = self.raw_block_prefix()
+        paths = [path for path in glob(f'{prefix}[0-9]*')
+                 if len(path) > len(prefix)
+                 and int(path[len(prefix):]) < min_height]
+        if paths:
+            for path in paths:
+                try:
+                    os.remove(path)
+                except FileNotFoundError:
+                    pass
+            self.logger.info(f'deleted {len(paths):,d} stale block files')
+
+    # -- UTXO database
+
+    def read_utxo_state(self) -> None:
+        state = self.utxo_db.get(self.DB_STATE_KEY)
+        if not state:
+            self.db_height = -1
+            self.db_tx_count = 0
+            self.db_tip = b'\0' * 32
+            self.db_version = max(self.DB_VERSIONS)
+            self.wall_time = 0
+            self.first_sync = True
+        else:
+            state = ast.literal_eval(state.decode())
+            if not isinstance(state, dict):
+                raise self.DBError('failed reading state from DB')
+            self.db_version = state['db_version']
+            if self.db_version not in self.DB_VERSIONS:
+                raise DBTooOldForMigrations(
+                    db_name="UTXO", db_version=self.db_version, supported_versions=self.DB_VERSIONS)
+            genesis_hash = state['genesis']
+            if genesis_hash != self.coin.GENESIS_HASH:
+                raise self.DBError(f'DB genesis hash {genesis_hash} does not '
+                                   f'match coin {self.coin.GENESIS_HASH}')
+            self.db_height = state['height']
+            self.db_tx_count = state['tx_count']
+            self.db_tip = state['tip']
+            self.wall_time = state['wall_time']
+            self.first_sync = state['first_sync']
+
+        # These are our state as we move ahead of DB state
+        self.fs_height = self.db_height
+        self.fs_tx_count = self.db_tx_count
+        self.last_flush_tx_count = self.fs_tx_count
+
+        # Upgrade DB
+        if self.db_version != max(self.DB_VERSIONS):
+            raise Exception("missing db upgrade")  # call future upgrade logic here
+
+        # Log some stats
+        self.logger.info(f'UTXO DB version: {self.db_version:d}')
+        self.logger.info(f'coin: {self.coin.NAME}')
+        self.logger.info(f'network: {self.coin.NET}')
+        self.logger.info(f'height: {self.db_height:,d}')
+        self.logger.info(f'tip: {hash_to_hex_str(self.db_tip)}')
+        self.logger.info(f'tx count: {self.db_tx_count:,d}')
+        if self.utxo_db.for_sync:
+            self.logger.info(f'flushing DB cache at {self.env.cache_MB:,d} MB')
+        if self.first_sync:
+            self.logger.info(
+                f'sync time so far: {util.formatted_time(self.wall_time)}'
+            )
+
+    def write_utxo_state(self, batch) -> None:
+        '''Write (UTXO) state to the batch.'''
+        state = {
+            'genesis': self.coin.GENESIS_HASH,
+            'height': self.db_height,
+            'tx_count': self.db_tx_count,
+            'tip': self.db_tip,
+            'wall_time': self.wall_time,
+            'first_sync': self.first_sync,
+            'db_version': self.db_version,
+        }
+        batch.put(self.DB_STATE_KEY, repr(state).encode())
+
+    async def all_utxos(self, hashX: bytes) -> Sequence[UTXO]:
+        '''Return all UTXOs for an address sorted in no particular order.'''
+        def read_utxos():
+            utxos = []  # type: list[UTXO]
+            utxos_append = utxos.append
+            # Key: b'u' + address_hashX + txout_idx + tx_num
+            # Value: the UTXO value as a 64-bit unsigned integer
+            prefix = b'u' + hashX
+            for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
+                txout_idx = unpack_txoutidx(db_key[-TXNUM_LEN-TXOUTIDX_LEN:-TXNUM_LEN])
+                tx_num = unpack_txnum(db_key[-TXNUM_LEN:])
+                value = unpack_satoshis_val(db_value)
+                txid_rev, height = self.fs_txid_rev(tx_num)
+                utxos_append(UTXO(tx_num, txout_idx, txid_rev, height, value))
+            return utxos
+
+        while True:
+            utxos = await run_in_thread(read_utxos)
+            if all(utxo.txid_rev is not None for utxo in utxos):
+                return utxos
+            self.logger.warning(f'all_utxos: tx hash not '
+                                f'found (reorg?), retrying...')
+            await sleep(0.25)
+
+    async def lookup_utxos(self, prevouts: Sequence[tuple[bytes, int]]) -> Sequence[Optional[tuple[bytes, int]]]:
+        '''For each prevout, lookup it up in the DB and return a (hashX,
+        value) pair or None if not found.
+
+        Used by the mempool code.
+        '''
+        def lookup_hashXs() -> Sequence[tuple[Optional[bytes], Optional[bytes]]]:
+            '''Return (hashX, suffix) pairs, or None if not found,
+            for each prevout.
+            '''
+            def lookup_hashX(txid_rev: bytes, tx_idx: int) -> tuple[Optional[bytes], Optional[bytes]]:
+                idx_packed = pack_txoutidx(tx_idx)
+
+                # Key: b'h' + compressed_tx_hash + tx_idx + tx_num
+                # Value: hashX
+                prefix = b'h' + txid_rev[:COMP_TXID_LEN] + idx_packed
+
+                # Find which entry, if any, the TX_HASH matches.
+                for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
+                    tx_num_packed = db_key[-TXNUM_LEN:]
+                    tx_num = unpack_txnum(tx_num_packed)
+                    hash, _height = self.fs_txid_rev(tx_num)
+                    if hash == txid_rev:
+                        return hashX, idx_packed + tx_num_packed
+                return None, None
+            return [lookup_hashX(*prevout) for prevout in prevouts]
+
+        def lookup_utxos(
+                hashX_pairs: Sequence[tuple[Optional[bytes], Optional[bytes]]],
+        ) -> Sequence[Optional[tuple[bytes, int]]]:
+            def lookup_utxo(hashX: bytes, suffix: bytes) -> Optional[tuple[bytes, int]]:
+                if not hashX:
+                    # This can happen when the daemon is a block ahead
+                    # of us and has mempool txs spending outputs from
+                    # that new block
+                    return None
+                # Key: b'u' + address_hashX + tx_idx + tx_num
+                # Value: the UTXO value as a 64-bit unsigned integer
+                key = b'u' + hashX + suffix
+                db_value = self.utxo_db.get(key)
+                if not db_value:
+                    # This can happen if the DB was updated between
+                    # getting the hashXs and getting the UTXOs
+                    return None
+                value = unpack_satoshis_val(db_value)
+                return hashX, value
+            return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
+
+        hashX_pairs = await run_in_thread(lookup_hashXs)
+        return await run_in_thread(lookup_utxos, hashX_pairs)

src/electrumx/server/db_util.py

@@ -0,0 +1,88 @@
+# Copyright (C) 2026 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from typing import Sequence
+
+from electrumx.lib.util import (
+    pack_be_uint32, unpack_be_uint32,
+    pack_be_uint64, unpack_be_uint64,
+)
+
+
+# txnum
+TXNUM_LEN = 5
+TXNUM_PADDING = bytes(8 - TXNUM_LEN)
+
+
+def unpack_txnum(tx_numb: bytes) -> int:
+    return unpack_be_uint64(TXNUM_PADDING + tx_numb)[0]
+
+
+def pack_txnum(tx_num: int) -> bytes:
+    return pack_be_uint64(tx_num)[-TXNUM_LEN:]
+
+
+assert pack_txnum(0x123) == bytes.fromhex("0000000123")
+assert unpack_txnum(pack_txnum(0x123)) == 0x123
+
+
+# txout_idx
+TXOUTIDX_LEN = 3
+TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN)
+
+
+def unpack_txoutidx(txout_idx: bytes) -> int:
+    return unpack_be_uint32(TXOUTIDX_PADDING + txout_idx)[0]
+
+
+def pack_txoutidx(txout_idx: int) -> bytes:
+    return pack_be_uint32(txout_idx)[-TXOUTIDX_LEN:]
+
+
+assert pack_txoutidx(0x123) == bytes.fromhex("000123")
+assert unpack_txoutidx(pack_txoutidx(0x123)) == 0x123
+
+
+# sats
+def unpack_satoshis_val(sats: bytes) -> int:
+    return unpack_be_uint64(sats)[0]
+
+
+def pack_satoshis_val(sats: int) -> bytes:
+    return pack_be_uint64(sats)
+
+
+# block height
+BHEIGHT_LEN = 4
+
+
+def unpack_block_height(bheight: bytes) -> int:
+    return unpack_be_uint32(bheight)[0]
+
+
+def pack_block_height(bheight: int) -> bytes:
+    return pack_be_uint32(bheight)
+
+
+# dynamic block header offsets
+DYN_HEADER_OFFSET_LEN = 8
+
+
+def unpack_dyn_header_offset(offset: bytes) -> int:
+    return unpack_be_uint64(offset)[0]
+
+
+def pack_dyn_header_offset(offset: int) -> bytes:
+    return pack_be_uint64(offset)
+
+
+class DBTooOldForMigrations(RuntimeError):
+    def __init__(self, *, db_name: str, db_version: int, supported_versions: Sequence[int]):
+        cmd = 'rm -rf DB_DIRECTORY/{hist,meta,utxo}'
+        super().__init__(
+            f'Your {db_name} DB version is {db_version} but this software only handles versions {supported_versions}. '
+            f'Manually delete your database (e.g. `{cmd}`, and start again. '
+            f'Then, your DB will be rebuilt from genesis, likely taking several hours. '
+            f"If you don't have time for this now, you can temporarily downgrade the software."
+        )

src/electrumx/server/env.py

@@ -0,0 +1,245 @@
+# Copyright (c) 2016, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Class for handling environment configuration and defaults.'''
+
+
+import re
+from ipaddress import IPv4Address, IPv6Address
+from typing import Type, Sequence
+
+from aiorpcx import Service, ServicePart
+
+import electrumx
+from electrumx.lib.coins import Coin
+from electrumx.lib.env_base import EnvBase
+
+
+class ServiceError(Exception):
+    pass
+
+
+class Env(EnvBase):
+    '''Wraps environment configuration. Optionally, accepts a Coin class
+       as first argument to have ElectrumX serve custom coins not part of
+       the standard distribution.
+    '''
+
+    # Peer discovery
+    PD_OFF, PD_SELF, PD_ON = ('OFF', 'SELF', 'ON')
+    SSL_PROTOCOLS = {'ssl', 'wss'}
+    KNOWN_PROTOCOLS = {'ssl', 'tcp', 'ws', 'wss', 'rpc'}
+
+    coin: Type[Coin]
+
+    def __init__(self, coin=None):
+        super().__init__()
+        self.obsolete(["MAX_SUBSCRIPTIONS", "MAX_SUBS", "MAX_SESSION_SUBS", "BANDWIDTH_LIMIT",
+                       "HOST", "TCP_PORT", "SSL_PORT", "RPC_HOST", "RPC_PORT", "REPORT_HOST",
+                       "REPORT_TCP_PORT", "REPORT_SSL_PORT", "REPORT_HOST_TOR",
+                       "REPORT_TCP_PORT_TOR", "REPORT_SSL_PORT_TOR"])
+
+        # Core items
+
+        self.db_dir = self.required('DB_DIRECTORY')
+        self.db_engine = self.db_engine_enum()
+        self.daemon_url = self.required('DAEMON_URL')
+        if coin is not None:
+            assert issubclass(coin, Coin)
+            self.coin = coin
+        else:
+            coin_name = self.default('COIN', 'Bitcoin').strip()
+            network = self.default('NET', 'mainnet').strip()
+            self.coin = Coin.lookup_coin_class(coin_name, network)
+
+        # Peer discovery
+
+        self.peer_discovery = self.peer_discovery_enum()
+        self.peer_announce = self.boolean('PEER_ANNOUNCE', True)
+        self.force_proxy = self.boolean('FORCE_PROXY', False)
+        self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
+        self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
+
+        # Misc
+        self.banner_file = self.default('BANNER_FILE', None)
+        self.tor_banner_file = self.default('TOR_BANNER_FILE',
+                                            self.banner_file)
+        self.anon_logs = self.boolean('ANON_LOGS', False)
+        self.log_sessions = self.integer('LOG_SESSIONS', 3600)
+        self.log_level = self.default('LOG_LEVEL', 'info').upper()
+        self.donation_address = self.default('DONATION_ADDRESS', '')
+        self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
+        self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
+        self.cache_MB = self.integer('CACHE_MB', 1200)
+        self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
+        self.daemon_poll_interval_blocks_msec = self.integer('DAEMON_POLL_INTERVAL_BLOCKS', 5000)
+        self.daemon_poll_interval_mempool_msec = self.integer('DAEMON_POLL_INTERVAL_MEMPOOL', 5000)
+
+        # Server limits to help prevent DoS
+
+        self.max_send = self.integer('MAX_SEND', self.coin.DEFAULT_MAX_SEND)
+        self.max_recv = self.integer('MAX_RECV', self.coin.DEFAULT_MAX_RECV)
+        self._try_raising_rlimit_nofile()  # do this before setting max_sessions
+        self.max_sessions = self.sane_max_sessions()
+        self.cost_soft_limit = self.integer('COST_SOFT_LIMIT', 1000)
+        self.cost_hard_limit = self.integer('COST_HARD_LIMIT', 10000)
+        self.bw_unit_cost = self.integer('BANDWIDTH_UNIT_COST', 5000)
+        self.initial_concurrent = self.integer('INITIAL_CONCURRENT', 10)
+        self.request_sleep = self.integer('REQUEST_SLEEP', 2500)
+        self.request_timeout = self.integer('REQUEST_TIMEOUT', 30)
+        self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
+        self.session_group_by_subnet_ipv4 = self.integer('SESSION_GROUP_BY_SUBNET_IPV4', 24)
+        self.session_group_by_subnet_ipv6 = self.integer('SESSION_GROUP_BY_SUBNET_IPV6', 48)
+        self._check_and_fix_cost_limits()
+
+        # Services last - uses some env vars above
+
+        self.services = self.services_to_run()
+        if {service.protocol for service in self.services}.intersection(self.SSL_PROTOCOLS):
+            self.ssl_certfile = self.required('SSL_CERTFILE')
+            self.ssl_keyfile = self.required('SSL_KEYFILE')
+        self.report_services = self.services_to_report()
+
+    def _try_raising_rlimit_nofile(self) -> None:
+        """Try raising the max num open file soft limit to the hard limit."""
+        try:
+            import resource
+        except ImportError:
+            return  # No resource module on Windows
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if soft >= hard:
+            return
+        try:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+        except (ValueError, OSError):
+            self.logger.warning(
+                f"error raising RLIMIT_NOFILE (ulimit -n) soft limit. Stuck with {soft}. "
+                f"You must ensure that electrumx has a large open file limit. See HOWTO in docs.")
+        else:
+            self.logger.info(
+                f"successfully raised RLIMIT_NOFILE (ulimit -n) soft limit from {soft} to {hard}")
+
+    def sane_max_sessions(self):
+        '''Return the maximum number of sessions to permit.  Normally this
+        is MAX_SESSIONS.  However, to prevent open file exhaustion, adjust
+        downwards if running with a small open file rlimit.'''
+        env_value = self.integer('MAX_SESSIONS', 1000)
+        # No resource module on Windows
+        try:
+            import resource
+            nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
+            # We give the DB 250 files; allow ElectrumX 100 for itself
+            value = max(0, min(env_value, nofile_limit - 350))
+            if value < env_value:
+                self.logger.warning(
+                    f'lowered maximum sessions from {env_value:,d} to '
+                    f'{value:,d} because your open file limit is '
+                    f'{nofile_limit:,d}'
+                )
+        except ImportError:
+            value = 512  # that is what returned by stdio's _getmaxstdio()
+        return value
+
+    def _check_and_fix_cost_limits(self):
+        if self.cost_hard_limit < self.cost_soft_limit:
+            raise self.Error(f"COST_HARD_LIMIT must be >= COST_SOFT_LIMIT. "
+                             f"got (COST_HARD_LIMIT={self.cost_hard_limit} "
+                             f"and COST_SOFT_LIMIT={self.cost_soft_limit})")
+        # hard limit should be strictly higher than soft limit (unless both are 0)
+        if self.cost_hard_limit == self.cost_soft_limit and self.cost_soft_limit > 0:
+            self.logger.info("found COST_HARD_LIMIT == COST_SOFT_LIMIT. "
+                             "bumping COST_HARD_LIMIT by 1.")
+            self.cost_hard_limit = self.cost_soft_limit + 1
+
+    def _parse_services(self, services_str, default_func) -> Sequence[Service]:
+        result = []
+        for service_str in services_str.split(','):
+            if not service_str:
+                continue
+            try:
+                service = Service.from_string(service_str, default_func=default_func)
+            except Exception as e:
+                raise ServiceError(f'"{service_str}" invalid: {e}') from None
+            if service.protocol not in self.KNOWN_PROTOCOLS:
+                raise ServiceError(f'"{service_str}" invalid: unknown protocol')
+            result.append(service)
+
+        # Find duplicate addresses
+        service_map = {service.address: [] for service in result}
+        for service in result:
+            service_map[service.address].append(service)
+        for address, services in service_map.items():
+            if len(services) > 1:
+                raise ServiceError(f'address {address} has multiple services')
+
+        return result
+
+    def services_to_run(self) -> Sequence[Service]:
+        def default_part(protocol, part):
+            return default_services.get(protocol, {}).get(part)
+
+        default_services = {protocol: {ServicePart.HOST: 'all_interfaces'}
+                            for protocol in self.KNOWN_PROTOCOLS}
+        default_services['rpc'] = {ServicePart.HOST: 'localhost', ServicePart.PORT: 8000}
+        services = self._parse_services(self.default('SERVICES', ''), default_part)
+
+        # Find onion hosts
+        for service in services:
+            if str(service.host).endswith('.onion'):
+                raise ServiceError(f'bad host for SERVICES: {service}')
+
+        return services
+
+    def services_to_report(self):
+        services = self._parse_services(self.default('REPORT_SERVICES', ''), None)
+
+        for service in services:
+            if service.protocol == 'rpc':
+                raise ServiceError(f'bad protocol for REPORT_SERVICES: {service.protocol}')
+            if isinstance(service.host, (IPv4Address, IPv6Address)):
+                ip_addr = service.host
+                if (ip_addr.is_multicast or ip_addr.is_unspecified or
+                        (ip_addr.is_private and self.peer_announce)):
+                    raise ServiceError(f'bad IP address for REPORT_SERVICES: {ip_addr}')
+            elif service.host.lower() == 'localhost':
+                raise ServiceError(f'bad host for REPORT_SERVICES: {service.host}')
+
+        return services
+
+    def peer_discovery_enum(self):
+        pd = self.default('PEER_DISCOVERY', 'on').strip().lower()
+        if pd in ('off', ''):
+            return self.PD_OFF
+        elif pd == 'self':
+            return self.PD_SELF
+        else:
+            return self.PD_ON
+
+    def db_engine_enum(self) -> str:
+        from electrumx.server.storage import list_db_engine_choices
+        choices = list_db_engine_choices()
+        db_eng = self.default('DB_ENGINE', None)
+
+        if db_eng is None:
+            raise self.Error(
+                f"required envvar DB_ENGINE not set.\n"
+                f"Choose one from: {choices}.\n"
+                f"The corresponding dependencies also need to be installed. See pip extras.\n"
+                f"In ElectrumX 1.x versions, the default was leveldb.\n"
+                f"Warning: It is not possible to switch back and forth, "
+                f"the on-disk DB formats are not compatible with each other: when switching, "
+                f"you have to resync from genesis.\n"
+                f"LevelDB was written with HDDs in mind. RocksDB is more modern and performs better (on an SSD)."
+            )
+
+        db_eng = db_eng.lower()
+        if db_eng in choices:
+            return db_eng
+        else:
+            raise self.Error(
+                f"bad value for envvar DB_ENGINE: {db_eng!r}. Choose one from: {choices}."
+            )

src/electrumx/server/history.py

@@ -0,0 +1,210 @@
+# Copyright (c) 2016-2018, Neil Booth
+# Copyright (c) 2017, the ElectrumX authors
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''History by script hash (address).'''
+
+import ast
+import bisect
+import time
+from array import array
+from collections import defaultdict
+from typing import TYPE_CHECKING, Type, Optional, Sequence, Iterable
+
+import electrumx.lib.util as util
+from electrumx.lib.hash import HASHX_LEN
+from electrumx.server.db_util import (
+    DBTooOldForMigrations,
+    pack_txnum, unpack_txnum, TXNUM_LEN,
+)
+
+if TYPE_CHECKING:
+    from electrumx.server.storage import Storage
+
+
+class History:
+
+    DB_VERSIONS = (4, )
+    DB_STATE_KEY = b'state\0\0'
+
+    db: Optional['Storage']
+
+    def __init__(self):
+        self.logger = util.class_logger(__name__, self.__class__.__name__)
+        self.unflushed = set()  # type: set[bytes]
+        self.hist_db_tx_count = 0
+        self.hist_db_tx_count_next = 0  # after next flush, next value for self.hist_db_tx_count
+        self.db_version = max(self.DB_VERSIONS)
+        self.upgrade_cursor = -1
+
+        # Key: b'H' + address_hashX + tx_num
+        # Value: <null>
+        self.db = None
+
+    def open_db(
+            self,
+            *,
+            db_class: Type['Storage'],
+            for_sync: bool,
+            utxo_db_tx_count: int,
+    ) -> None:
+        self.db = db_class('hist', for_sync)
+        self.read_state()
+        self.clear_excess(utxo_db_tx_count)
+
+    def close_db(self) -> None:
+        if self.db:
+            self.db.close()
+            self.db = None
+
+    def read_state(self) -> None:
+        state = self.db.get(self.DB_STATE_KEY)
+        if state:
+            state = ast.literal_eval(state.decode())
+            if not isinstance(state, dict):
+                raise RuntimeError('failed reading state from history DB')
+            self.db_version = state.get('db_version', 0)
+            self.upgrade_cursor = state.get('upgrade_cursor', -1)
+            self.hist_db_tx_count = state.get('hist_db_tx_count', 0)
+            self.hist_db_tx_count_next = self.hist_db_tx_count
+
+        if self.db_version not in self.DB_VERSIONS:
+            raise DBTooOldForMigrations(
+                db_name="history", db_version=self.db_version, supported_versions=self.DB_VERSIONS)
+        if self.db_version != max(self.DB_VERSIONS):
+            raise Exception("missing db upgrade")  # call future upgrade logic here
+        self.logger.info(f'history DB version: {self.db_version}')
+
+    def clear_excess(self, utxo_db_tx_count: int) -> None:
+        # self.hist_db_tx_count != utxo_db_tx_count might happen as
+        # both DBs cannot be updated atomically.
+        # However, we MUST maintain the invariant that hist_db_tx_count >= utxo_db_tx_count:
+        # - when advancing blocks, hist_db is flushed first, so its count can be higher;
+        #   but when backing up (during reorg), hist_db is flushed last, so its count can be higher.
+        if self.hist_db_tx_count == utxo_db_tx_count:
+            return  # happy path
+        elif self.hist_db_tx_count < utxo_db_tx_count:
+            raise RuntimeError(f"corrupted DB. {self.hist_db_tx_count=} < {utxo_db_tx_count=}")
+
+        assert self.hist_db_tx_count > utxo_db_tx_count
+        self.logger.info(
+            'DB shut down uncleanly. Scanning for excess history flushes... '
+            'This might take a loooong time. :/'
+        )
+
+        utxo_db_tx_count_bytes = pack_txnum(utxo_db_tx_count)
+        keys = []  # FIXME could grow large in-memory?
+        for iter_cnt, (db_key, _db_val) in enumerate(self.db.iterator(prefix=b'H')):
+            if iter_cnt % 10_000_000 == 0:
+                self.logger.info(
+                    f'Scanning for excess history flushes... num history entries checked: '
+                    f'{iter_cnt // 1_000_000} million. will delete: {len(keys):,d}')
+            tx_numb = db_key[-TXNUM_LEN:]
+            # we exploit that pack_txnum uses big-endian byte-order, and just compare the bytes directly:
+            # tx_num = unpack_txnum(tx_numb)
+            if tx_numb >= utxo_db_tx_count_bytes:
+                keys.append(db_key)
+
+        self.logger.info(f'deleting {len(keys):,d} history entries')
+
+        self.hist_db_tx_count = utxo_db_tx_count
+        self.hist_db_tx_count_next = self.hist_db_tx_count
+        with self.db.write_batch() as batch:
+            for key in keys:
+                batch.delete(key)
+            self._write_state(batch)
+
+        self.logger.info('deleted excess history entries')
+
+    def _write_state(self, batch) -> None:
+        '''Write state to the history DB.'''
+        state = {
+            'hist_db_tx_count': self.hist_db_tx_count,
+            'db_version': self.db_version,
+            'upgrade_cursor': self.upgrade_cursor,
+        }
+        batch.put(self.DB_STATE_KEY, repr(state).encode())
+
+    def add_unflushed(
+            self,
+            hashXs_by_tx: list[list[bytes]],
+            first_tx_num: int,
+    ) -> None:
+        """This method must be thread-safe. (runs on bp.pool_executor)"""
+        unflushed = self.unflushed
+        for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num):
+            tx_numb = pack_txnum(tx_num)
+            for hashX in hashXs:
+                unflushed.add(hashX + tx_numb)
+
+    def update_tx_count_next(self, tx_count: int) -> None:
+        self.hist_db_tx_count_next = tx_count
+
+    def unflushed_memsize(self) -> int:
+        # note: the magic numbers here were estimated using util.deep_getsizeof
+        return len(self.unflushed) * 85 + len(self.unflushed) * (HASHX_LEN + TXNUM_LEN)
+
+    def assert_flushed(self) -> None:
+        assert not self.unflushed
+
+    def flush(self) -> None:
+        start_time = time.monotonic()
+        unflushed = self.unflushed
+
+        with self.db.write_batch() as batch:
+            for hashX_txnum in sorted(unflushed):
+                db_key = b'H' + hashX_txnum
+                batch.put(db_key, b'')
+            self.hist_db_tx_count = self.hist_db_tx_count_next
+            self._write_state(batch)
+
+        count = len(unflushed)
+        unflushed.clear()
+
+        if self.db.for_sync:
+            elapsed = time.monotonic() - start_time
+            self.logger.info(f'flushed history in {elapsed:.1f}s '
+                             f'for {count:,d} history entries')
+
+    def backup(self, *, hashXs: Iterable[bytes], tx_count: int) -> None:
+        self.assert_flushed()
+        nremoves = 0
+        with self.db.write_batch() as batch:
+            for hashX in sorted(hashXs):
+                deletes = []
+                prefix = b'H' + hashX
+                for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True):
+                    tx_numb = db_key[-TXNUM_LEN:]
+                    tx_num = unpack_txnum(tx_numb)
+                    if tx_num >= tx_count:
+                        nremoves += 1
+                        deletes.append(db_key)
+                    else:
+                        # note: we can break now, due to 'reverse=True' and txnums being big endian
+                        break
+                for key in deletes:
+                    batch.delete(key)
+            self.hist_db_tx_count = tx_count
+            self.hist_db_tx_count_next = self.hist_db_tx_count
+            self._write_state(batch)
+
+        self.logger.info(f'backing up removed {nremoves:,d} history entries')
+
+    def get_txnums(self, hashX: bytes, limit: int = 1000) -> Iterable[int]:
+        '''Generator that returns an unpruned, sorted list of tx_nums in the
+        history of a hashX.  Includes both spending and receiving
+        transactions.  By default yields at most 1000 entries.  Set
+        limit to None to get them all.  '''
+        limit = util.resolve_limit(limit)
+        prefix = b'H' + hashX
+        for db_key, db_val in self.db.iterator(prefix=prefix):
+            tx_numb = db_key[-TXNUM_LEN:]
+            if limit == 0:
+                return
+            tx_num = unpack_txnum(tx_numb)
+            yield tx_num
+            limit -= 1

src/electrumx/server/mempool.py

@@ -0,0 +1,626 @@
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Mempool handling.'''
+
+import itertools
+import time
+from abc import ABC, abstractmethod
+import asyncio
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set, Iterable
+import math
+from graphlib import TopologicalSorter
+
+from aiorpcx import run_in_thread, sleep, ignore_after
+
+from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash
+from electrumx.lib.tx import SkipTxDeserialize
+from electrumx.lib.util import class_logger, chunks, OldTaskGroup, LogTimeTaken
+from electrumx.lib.tx import TXOSpendStatus, TxOutpoint
+from electrumx.server.db import UTXO
+
+if TYPE_CHECKING:
+    from electrumx.lib.coins import Coin
+
+
+DB_UTXO_MAP = dict[TxOutpoint, Optional[tuple[bytes, int]]]  # prevout->(hashX,value_in_sats)
+
+
+@dataclass(slots=True)
+class MemPoolTx:
+    prevouts: Sequence[TxOutpoint]  # (txid_rev, txout_idx)
+    # A pair is a (hashX, value) tuple
+    in_pairs: Optional[Sequence[tuple[bytes, int]]]  # (hashX, value_in_sats)
+    out_pairs: Sequence[tuple[bytes, int]]  # (hashX, value_in_sats)
+    fee: int  # in sats
+    size: int  # in vbytes
+
+
+@dataclass(slots=True)
+class MemPoolTxSummary:
+    txid_rev: bytes
+    fee: int  # in sats
+    has_unconfirmed_inputs: bool
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class RecentMemPoolTx:
+    txid_rev: bytes
+    fee: int         # in sats
+    vsize: int       # in vbytes
+
+
+class DBSyncError(Exception):
+    pass
+
+
+class MemPoolAPI(ABC):
+    '''A concrete instance of this class is passed to the MemPool object
+    and used by it to query DB and blockchain state.'''
+
+    @abstractmethod
+    async def height(self) -> int:
+        '''Query bitcoind for its height.'''
+
+    @abstractmethod
+    def cached_height(self) -> Optional[int]:
+        '''Return the height of bitcoind the last time it was queried,
+        for any reason, without actually querying it.
+        '''
+
+    @abstractmethod
+    async def daemon_height_changed(self) -> None:
+        '''Wait until bitcoind's height changes.'''
+
+    @abstractmethod
+    def db_height(self) -> int:
+        '''Return the height flushed to the on-disk DB.'''
+
+    @abstractmethod
+    async def db_height_changed(self) -> None:
+        '''Wait until the on-disk DB height changes.'''
+
+    @abstractmethod
+    async def mempool_txids_hum(self) -> Sequence[str]:
+        '''Query bitcoind for the txids of all transactions in its
+        mempool, returned as a list.'''
+
+    @abstractmethod
+    async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]:
+        '''Query bitcoind for the serialized raw transactions with the given
+        txids.  Missing transactions are returned as None.
+
+        txids_hum is an iterable of hexadecimal hash strings.'''
+
+    @abstractmethod
+    async def lookup_utxos(self, prevouts: Sequence[TxOutpoint]) -> Sequence[Optional[Tuple[bytes, int]]]:
+        '''Return a list of (hashX, value) pairs, one for each prevout if unspent,
+        otherwise return None if spent or not found (for the given prevout).
+
+        prevouts - an iterable of (txid_rev, txout_idx) pairs
+        '''
+
+    @abstractmethod
+    async def on_mempool(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        '''Called each time the mempool is synchronized.  touched_hashxs and
+        touched_outpoints are sets of hashXs and tx outpoints touched since
+        the previous call.  height is the  daemon's height at the time the
+        mempool was obtained.
+        '''
+
+
+class MemPool:
+    '''Representation of the daemon's mempool.
+
+        coin - a coin class from coins.py
+        api - an object implementing MemPoolAPI
+
+    Updated regularly in caught-up state.  Goal is to enable efficient
+    response to the calls in the external interface.  To that end we
+    maintain the following maps:
+
+       tx:     txid_rev -> MemPoolTx
+       hashXs: hashX   -> set of all txids_rev of txs touching the hashX
+    '''
+
+    def __init__(
+            self,
+            coin: Type['Coin'],
+            api: MemPoolAPI,
+            *,
+            refresh_secs=5.0,
+            log_status_secs=60.0,
+    ):
+        assert isinstance(api, MemPoolAPI)
+        self.coin = coin
+        self.api = api
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.txs = {}  # type: Dict[bytes, MemPoolTx]  # txid_rev->tx
+        self.hashXs = defaultdict(set)  # type: Dict[Optional[bytes], Set[bytes]]  # hashX->txids_rev
+        self.txo_to_spender = {}  # type: Dict[TxOutpoint, bytes]  # prevout->txid_rev
+        self.cached_compact_histogram = []  # type: Sequence[tuple[float, int]]
+        self.refresh_secs = refresh_secs
+        self.log_status_secs = log_status_secs
+        # Prevents mempool refreshes during fee histogram calculation
+        self.lock = asyncio.Lock()
+
+    async def _logging(self, synchronized_event):
+        '''Print regular logs of mempool stats.'''
+        self.logger.info('beginning processing of daemon mempool.  '
+                         'This can take some time...')
+        start = time.monotonic()
+        await synchronized_event.wait()
+        elapsed = time.monotonic() - start
+        self.logger.info(f'synced in {elapsed:.2f}s')
+        while True:
+            mempool_size = sum(tx.size for tx in self.txs.values()) / 1_000_000
+            self.logger.info(f'{len(self.txs):,d} txs {mempool_size:.2f} MB, '
+                             f'touching {len(self.hashXs):,d} addresses. '
+                             f'{len(self.txo_to_spender):,d} spends.')
+            await sleep(self.log_status_secs)
+            await synchronized_event.wait()
+
+    async def _refresh_histogram(self, synchronized_event):
+        while True:
+            await synchronized_event.wait()
+            async with self.lock:
+                # Threaded as can be expensive
+                bin_size = self.coin.MEMPOOL_COMPACT_HISTOGRAM_BINSIZE
+                await run_in_thread(self._update_histogram, bin_size)
+            await sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
+
+    def _update_histogram(self, bin_size):
+        # Build a histogram by fee rate
+        t0 = time.monotonic()
+        histogram = defaultdict(int)
+        for tx in self.txs.values():
+            fee_rate = tx.fee / tx.size
+            # use 0.1 sat/byte resolution
+            # note: rounding *down* is intentional. This ensures txs
+            #       with a given fee rate will end up counted in the expected
+            #       bucket/interval of the compact histogram.
+            fee_rate = math.floor(10 * fee_rate) / 10
+            histogram[fee_rate] += tx.size
+
+        compact = self._compress_histogram(histogram, bin_size=bin_size)
+        self.logger.info(f'compact fee histogram: {compact}. took {time.monotonic() - t0:.3f}s.')
+        self.cached_compact_histogram = compact
+
+    @classmethod
+    def _compress_histogram(
+            cls, histogram: Dict[float, int], *, bin_size: int
+    ) -> Sequence[Tuple[float, int]]:
+        '''Calculate and return a compact fee histogram as needed for
+        "mempool.get_fee_histogram" protocol request.
+
+        histogram: feerate (sat/vbyte) -> total size in bytes of txs that pay approx feerate
+        bin_size: ~minimum vsize of a bucket of txs in the result (e.g. 100 kb)
+        '''
+        # Now compact it.  For efficiency, get_fees returns a
+        # compact histogram with variable bin size.  The compact
+        # histogram is an array of (fee_rate, vsize) values.
+        # vsize_n is the cumulative virtual size of mempool
+        # transactions with a fee rate in the interval
+        # [rate_(n-1), rate_n)], and rate_(n-1) > rate_n.
+        # Intervals are chosen to create tranches containing at
+        # least a certain cumulative size (bin_size) of transactions.
+        assert bin_size > 0
+        compact = []
+        cum_size = 0
+        prev_fee_rate = None
+        for fee_rate, size in sorted(histogram.items(), reverse=True):
+            # if there is a big lump of txns at this specific size,
+            # consider adding the previous item now (if not added already)
+            if size > 2 * bin_size and prev_fee_rate is not None and cum_size > 0:
+                compact.append((prev_fee_rate, cum_size))
+                cum_size = 0
+                bin_size *= 1.1
+            # now consider adding this item
+            cum_size += size
+            if cum_size > bin_size:
+                compact.append((fee_rate, cum_size))
+                cum_size = 0
+                bin_size *= 1.1
+            prev_fee_rate = fee_rate
+        return compact
+
+    def _accept_transactions(
+            self,
+            *,
+            tx_map: Dict[bytes, MemPoolTx],  # txid_rev->tx
+            utxo_map: DB_UTXO_MAP,  # prevout->(hashX,value_in_sats)
+            touched_hashxs: Set[bytes],  # set of hashXs
+            touched_outpoints: Set[TxOutpoint],  # set of outpoints
+            topologically_sort: bool,
+    ) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]:
+        '''Accept transactions in tx_map to the mempool if all their inputs
+        can be found in the existing mempool or a utxo_map from the
+        DB.
+
+        Returns an (unprocessed tx_map, unspent utxo_map) pair.
+        '''
+        hashXs = self.hashXs
+        txs = self.txs
+        txo_to_spender = self.txo_to_spender
+
+        if topologically_sort:
+            # Sort tx_map so that parent txs come first (in case of unconf chain).
+            # This is just an optimization so that fewer txs will get "deferred", leading to better perf.
+            # Dependencies outside tx_map (already in chainstate (utxo_map), or our mempool (self.txs),
+            # or MISSING) are ignored.
+            cand_to_parents = {
+                txid: {parent_txid for (parent_txid, txout_idx) in tx.prevouts
+                       if parent_txid in tx_map}
+                for (txid, tx) in tx_map.items()}
+            cand_txids = list(TopologicalSorter(cand_to_parents).static_order())
+            assert len(tx_map) == len(cand_to_parents) == len(cand_txids), \
+                f"{len(tx_map)=}, {len(cand_to_parents)=}, {len(cand_txids)=}"
+            del cand_to_parents
+        else:
+            cand_txids = tx_map.keys()
+
+        deferred = {}  # type: dict[bytes, MemPoolTx]
+        unspent = set(utxo_map)
+        # Try to find all prevouts so we can accept the candidate TXs from tx_map into our mempool
+        for txid_rev in cand_txids:
+            tx = tx_map[txid_rev]
+            in_pairs = []
+            try:
+                for prevout in tx.prevouts:
+                    # first, look for parent tx among confirmed UTXOs:
+                    utxo = utxo_map.get(prevout)
+                    if not utxo:  # second, look for parent tx in mempool
+                        prev_hash, prev_index = prevout
+                        # Raises KeyError if prev_hash is not in txs
+                        utxo = txs[prev_hash].out_pairs[prev_index]
+                    in_pairs.append(utxo)
+            except KeyError:
+                deferred[txid_rev] = tx
+                continue
+
+            # Spend the prevouts
+            unspent.difference_update(tx.prevouts)
+
+            # Save the in_pairs, compute the fee and accept the TX
+            tx.in_pairs = tuple(in_pairs)
+            # Avoid negative fees if dealing with generation-like transactions
+            # because some in_parts would be missing
+            tx.fee = max(0, (sum(v for _, v in tx.in_pairs) -
+                             sum(v for _, v in tx.out_pairs)))
+            txs[txid_rev] = tx
+
+            for hashX, _value in itertools.chain(tx.in_pairs, tx.out_pairs):
+                touched_hashxs.add(hashX)
+                hashXs[hashX].add(txid_rev)
+            for prevout in tx.prevouts:
+                txo_to_spender[prevout] = txid_rev
+                touched_outpoints.add(prevout)
+            for out_idx, out_pair in enumerate(tx.out_pairs):
+                touched_outpoints.add((txid_rev, out_idx))
+
+        return deferred, {prevout: utxo_map[prevout] for prevout in unspent}
+
+    async def _refresh_hashes(self, synchronized_event):
+        '''Refresh our view of the daemon's mempool.'''
+        # touched_* accumulates between calls to on_mempool and each
+        # call transfers ownership
+        touched_hashxs = set()
+        touched_outpoints = set()
+        prev_mempool_height = -1
+        while True:
+            height = self.api.cached_height()
+            is_new_block = height != prev_mempool_height
+            with LogTimeTaken(self.logger, "getrawmempool RPC", enabled=is_new_block):
+                txids_hum = await self.api.mempool_txids_hum()
+            if height != await self.api.height():  # if height changed *again*, re-start
+                continue
+            with LogTimeTaken(self.logger, "convert_txids", enabled=is_new_block):
+                txids_rev = await run_in_thread(lambda: {hex_str_to_hash(hh) for hh in txids_hum})
+            if self.api.db_height() < height:
+                # DB not yet caught up. give it some time...
+                async with ignore_after(self.refresh_secs):
+                    await self.api.db_height_changed()
+            try:
+                async with self.lock:
+                    await self._process_mempool(
+                        all_txids_rev=txids_rev,
+                        touched_hashxs=touched_hashxs,
+                        touched_outpoints=touched_outpoints,
+                        mempool_height=height,
+                        debug_logs=is_new_block,
+                    )
+            except DBSyncError:
+                # The UTXO DB is not at the same height as the
+                # mempool; wait and try again
+                self.logger.debug('waiting for DB to sync')
+            else:
+                synchronized_event.set()
+                synchronized_event.clear()
+                await self.api.on_mempool(
+                    touched_hashxs=touched_hashxs,
+                    touched_outpoints=touched_outpoints,
+                    height=height,
+                )
+                prev_mempool_height = height
+                touched_hashxs = set()
+                touched_outpoints = set()
+            # poll bitcoind's whole mempool every few seconds; or instantly after the DB height changes
+            async with ignore_after(self.refresh_secs):
+                async with OldTaskGroup(wait=any) as group:
+                    await group.spawn(self.api.db_height_changed())
+                    await group.spawn(self.api.daemon_height_changed())
+
+    async def _process_mempool(
+            self,
+            *,
+            all_txids_rev: Set[bytes],  # set of txids_rev  # complete view of daemon's mempool
+            touched_hashxs: Set[bytes],  # set of hashXs
+            touched_outpoints: Set[TxOutpoint],  # set of outpoints
+            mempool_height: int,
+            debug_logs: bool = False,
+    ) -> None:
+        # Re-sync with the new set of hashes
+        txs = self.txs
+        hashXs = self.hashXs
+        txo_to_spender = self.txo_to_spender
+
+        if mempool_height != self.api.db_height():  # FIXME should compare blockhash
+            raise DBSyncError
+
+        # 1. Handle txs that have disappeared (evicted, just got mined, etc)
+        # TODO split disappeared txs workload into a threadpool, chunks of ~200 txs
+        def handle_disappeared_txs() -> None:
+            nonlocal touched_hashxs
+            for txid_rev in disappeared_hashes:
+                tx = txs.pop(txid_rev)
+                # hashXs
+                tx_hashXs = {hashX for hashX, value in tx.in_pairs}
+                tx_hashXs.update(hashX for hashX, value in tx.out_pairs)
+                for hashX in tx_hashXs:
+                    hashXs[hashX].remove(txid_rev)
+                    if not hashXs[hashX]:
+                        del hashXs[hashX]
+                touched_hashxs |= tx_hashXs
+                # outpoints
+                for prevout in tx.prevouts:
+                    del txo_to_spender[prevout]
+                    touched_outpoints.add(prevout)
+                for out_idx, out_pair in enumerate(tx.out_pairs):
+                    touched_outpoints.add((txid_rev, out_idx))
+
+        with LogTimeTaken(self.logger, "_process_mempool() removing txs", enabled=debug_logs):
+            disappeared_hashes = await run_in_thread(lambda: set(txs) - all_txids_rev)
+            handle_disappeared_txs()
+
+        # 2. Process new transactions
+        t0 = time.monotonic()
+        new_hashes = await run_in_thread(lambda: list(all_txids_rev.difference(txs)))
+        if new_hashes:
+            # 2.1. fetch raw txs from bitcoin daemon
+            group = OldTaskGroup()
+            for hashes in chunks(new_hashes, 200):
+                coro = self._fetch_raw_txs_and_utxos(
+                    new_txids_rev=hashes,
+                    all_txids_rev=all_txids_rev,
+                )
+                await group.spawn(coro)
+            if mempool_height != self.api.db_height():
+                raise DBSyncError
+
+            tx_map = {}  # type: dict[bytes, MemPoolTx]
+            utxo_map = {}  # type: DB_UTXO_MAP
+            async for task in group:
+                partial_tx_map, partial_utxo_map = task.result()
+                tx_map.update(partial_tx_map)
+                utxo_map.update(partial_utxo_map)
+            if debug_logs:
+                self.logger.debug(f'_process_mempool() fetching txs took {time.monotonic() - t0:.3f}s. {len(new_hashes)=}')
+
+            # 2.2. accept txs into our mempool
+            def accept_txs_loop() -> None:
+                # Accept candidate txs from tx_map into our mempool.
+                # We only accept candidates for which we can find a UTXO for each tx input:
+                # - some UTXOs we find in the DB=utxo_map (chainstate),
+                # - some we find in self.txs (our memool),
+                # - some we might only find in tx_map (among the candidates: consider long chain of unconfirmed txs)
+                # - some we might not find anywhere: if DB is corrupted or bitcoind gave inconsistent mempool, or other race
+                # In each iteration, we loop over tx_map and move accepted transactions from it to self.txs.
+                # In the worst degenerate case, this could be O(n^2). To avoid that, we topologically sort tx_map.
+                nonlocal tx_map, utxo_map
+                prior_txmap_size = 0
+                while tx_map and len(tx_map) != prior_txmap_size:
+                    prior_txmap_size = len(tx_map)
+                    tx_map, utxo_map = self._accept_transactions(
+                        tx_map=tx_map,
+                        utxo_map=utxo_map,
+                        touched_hashxs=touched_hashxs,
+                        touched_outpoints=touched_outpoints,
+                        topologically_sort=True,
+                    )
+                if tx_map:
+                    self.logger.error(f'{len(tx_map)} txs dropped')
+
+            with LogTimeTaken(self.logger, f"_process_mempool() accepting txs ({len(new_hashes)=})", enabled=debug_logs):
+                accept_txs_loop()
+
+    async def _fetch_raw_txs_and_utxos(
+            self,
+            *,
+            new_txids_rev: set[bytes],  # (some) new candidate txs for our mempool
+            all_txids_rev: set[bytes],  # complete view of daemon's mempool
+    ) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]:
+        '''Fetch a list of mempool transactions, and lookup corresponding UTXOs in the DB.'''
+        txids_hum_iter = (hash_to_hex_str(hash) for hash in new_txids_rev)
+        raw_txs = await self.api.raw_transactions(txids_hum_iter)
+
+        def deserialize_txs() -> Dict[bytes, MemPoolTx]:
+            """This function is pure"""
+            to_hashX = self.coin.hashX_from_script
+            deserializer = self.coin.DESERIALIZER
+
+            txs = {}  # type: Dict[bytes, MemPoolTx]
+            for txid_rev, raw_tx in zip(new_txids_rev, raw_txs):
+                # The daemon may have evicted the tx from its
+                # mempool or it may have gotten in a block
+                if not raw_tx:
+                    continue
+                try:
+                    tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
+                except SkipTxDeserialize as ex:
+                    self.logger.debug(f'skipping tx {hash_to_hex_str(txid_rev)}: {ex}')
+                    continue
+                # Convert the inputs and outputs into (hashX, value) pairs
+                # Drop generation-like inputs from MemPoolTx.prevouts
+                txin_pairs = tuple((txin.prev_txid_rev, txin.prev_idx)
+                                   for txin in tx.inputs
+                                   if not txin.is_generation())
+                txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
+                                    for txout in tx.outputs)
+                txs[txid_rev] = MemPoolTx(
+                    prevouts=txin_pairs,
+                    in_pairs=None,
+                    out_pairs=txout_pairs,
+                    fee=0,
+                    size=tx_size,
+                )
+            return txs
+
+        # Thread this potentially slow operation so as not to block
+        tx_map = await run_in_thread(deserialize_txs)  # type: Dict[bytes, MemPoolTx]
+
+        # Determine all prevouts not in the mempool, and fetch the
+        # UTXO information from the database.  Failed prevout lookups
+        # return None - concurrent database updates happen - which is
+        # relied upon by _accept_transactions. Ignore prevouts that are
+        # generation-like.
+        prevouts = tuple(prevout for tx in tx_map.values()
+                         for prevout in tx.prevouts
+                         if prevout[0] not in all_txids_rev)
+        utxos = await self.api.lookup_utxos(prevouts)
+        utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)}
+
+        return tx_map, utxo_map
+
+    #
+    # External interface
+    #
+
+    async def keep_synchronized(self, synchronized_event: asyncio.Event) -> None:
+        '''Keep the mempool synchronized with the daemon.'''
+        async with OldTaskGroup() as group:
+            await group.spawn(self._refresh_hashes(synchronized_event))
+            await group.spawn(self._refresh_histogram(synchronized_event))
+            await group.spawn(self._logging(synchronized_event))
+
+    async def balance_delta(self, hashX: bytes) -> int:
+        '''Return the unconfirmed amount in the mempool for hashX.
+
+        Can be positive or negative.
+        '''
+        value = 0
+        if hashX in self.hashXs:
+            for txid_rev in self.hashXs[hashX]:
+                tx = self.txs[txid_rev]
+                value -= sum(v for h168, v in tx.in_pairs if h168 == hashX)
+                value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
+        return value
+
+    async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]:
+        '''Return a compact fee histogram of the current mempool.'''
+        return self.cached_compact_histogram
+
+    async def potential_spends(self, hashX: bytes) -> set[TxOutpoint]:
+        '''Return a set of (prev_hash, prev_idx) pairs from mempool
+        transactions that touch hashX.
+
+        None, some or all of these may be spends of the hashX, but all
+        actual spends of it (in the DB or mempool) will be included.
+        '''
+        result = set()
+        for txid_rev in self.hashXs.get(hashX, ()):
+            tx = self.txs[txid_rev]
+            result.update(tx.prevouts)
+        return result
+
+    async def transaction_summaries(self, hashX: bytes) -> Sequence[MemPoolTxSummary]:
+        '''Return a list of MemPoolTxSummary objects for the hashX,
+        sorted as expected by protocol methods.
+        '''
+        result = []  # type: list[MemPoolTxSummary]
+        for txid_rev in self.hashXs.get(hashX, ()):
+            tx = self.txs[txid_rev]
+            has_ui = any(hash in self.txs for hash, idx in tx.prevouts)
+            result.append(MemPoolTxSummary(txid_rev, tx.fee, has_ui))
+        result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.txid_rev[::-1]))
+        return result
+
+    async def unordered_UTXOs(self, hashX: bytes) -> Sequence[UTXO]:
+        '''Return an unordered list of UTXO named tuples from mempool
+        transactions that pay to hashX.
+
+        This does not consider if any other mempool transactions spend
+        the outputs.
+        '''
+        utxos = []
+        for txid_rev in self.hashXs.get(hashX, ()):
+            tx = self.txs.get(txid_rev)
+            for pos, (hX, value) in enumerate(tx.out_pairs):
+                if hX == hashX:
+                    utxos.append(UTXO(-1, pos, txid_rev, 0, value))
+        return utxos
+
+    async def spender_for_txo(self, prev_txid_rev: bytes, txout_idx: int) -> 'TXOSpendStatus':
+        '''For an outpoint, returns its spend-status.
+        This only considers the mempool, not the DB/blockchain, so e.g. mined
+        txs are not distinguished from txs that never existed.
+        '''
+        # look up funding tx
+        prev_tx = self.txs.get(prev_txid_rev, None)
+        if prev_tx is None:
+            # funding tx already mined or never existed
+            funder_height = None
+        else:
+            if len(prev_tx.out_pairs) <= txout_idx:
+                # output idx out of bounds...?
+                return TXOSpendStatus(funder_height=None)
+            prev_has_ui = any(hash in self.txs for hash, idx in prev_tx.prevouts)
+            funder_height = -prev_has_ui
+        prevout = (prev_txid_rev, txout_idx)
+        # look up spending tx
+        spender_txid_rev = self.txo_to_spender.get(prevout, None)
+        if spender_txid_rev is None:
+            return TXOSpendStatus(funder_height=funder_height)
+        spender_tx = self.txs.get(spender_txid_rev, None)
+        if spender_tx is None:
+            self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txid_rev)} not in"
+                                f"mempool, but txo_to_spender referenced it as spender "
+                                f"of {hash_to_hex_str(prev_txid_rev)}:{txout_idx} ?!")
+            return TXOSpendStatus(funder_height=funder_height)
+        spender_has_ui = any(txid_rev in self.txs for txid_rev, idx in spender_tx.prevouts)
+        spender_height = -spender_has_ui
+        return TXOSpendStatus(
+            funder_height=funder_height,
+            spender_txid_rev=spender_txid_rev,
+            spender_height=spender_height,
+        )
+
+    async def get_recently_added_txs(self, *, count: int) -> Sequence[RecentMemPoolTx]:
+        # note: inefficient for large "count"s
+        it = reversed(self.txs.items())
+        count = min(count, len(self.txs))
+        mempool_txs = [next(it) for _ in range(count)]
+        return [
+            RecentMemPoolTx(txid_rev=hash, fee=mtx.fee, vsize=mtx.size)
+            for hash, mtx in mempool_txs]

src/electrumx/server/peers.py

@@ -0,0 +1,634 @@
+# Copyright (c) 2017-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Peer management.'''
+
+import asyncio
+import random
+import socket
+import ssl
+import time
+from collections import Counter, defaultdict
+from ipaddress import IPv4Address, IPv6Address
+from typing import TYPE_CHECKING, Type, Any, Optional
+from functools import partial
+
+import aiohttp
+from aiorpcx import (Event, Notification, RPCSession, SOCKSError,
+                     SOCKSProxy, TaskTimeout, connect_rs,
+                     handler_invocation, ignore_after, sleep)
+from aiorpcx.jsonrpc import CodeMessageError
+
+from electrumx.lib.peer import Peer
+from electrumx.lib.util import class_logger, json_deserialize, OldTaskGroup
+from electrumx.server.session import RPCSessionWithTaskGroup
+from electrumx.server.transport import PaddedRSTransport
+
+if TYPE_CHECKING:
+    from electrumx.server.env import Env
+    from electrumx.server.db import DB
+
+
+PEER_GOOD, PEER_STALE, PEER_NEVER, PEER_BAD = range(4)
+STATUS_DESCS = ('good', 'stale', 'never', 'bad')
+STALE_SECS = 3 * 3600
+WAKEUP_SECS = 300
+PEER_ADD_PAUSE = 600
+
+
+class BadPeerError(Exception):
+    pass
+
+
+def assert_good(message, result, instance):
+    if not isinstance(result, instance):
+        raise BadPeerError(f'{message} returned bad result type '
+                           f'{type(result).__name__}')
+
+
+class PeerSession(RPCSessionWithTaskGroup):
+    '''An outgoing session to a peer.'''
+
+    async def handle_request(self, request):
+        # We subscribe so might be unlucky enough to get a notification...
+        if (isinstance(request, Notification) and
+                request.method == 'blockchain.headers.subscribe'):
+            pass
+        else:
+            await handler_invocation(None, request)   # Raises
+
+
+class PeerManager:
+    '''Looks after the DB of peer network servers.
+
+    Attempts to maintain a connection with up to 8 peers.
+    Issues a 'peers.subscribe' RPC to them and tells them our data.
+    '''
+    def __init__(self, env: 'Env', db: 'DB'):
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        # Initialise the Peer class
+        Peer.DEFAULT_PORTS = env.coin.PEER_DEFAULT_PORTS
+        self.env = env
+        self.db = db
+
+        # Our reported clearnet and Tor Peers, if any
+        sclass = env.coin.SESSIONCLS
+        self.myselves = [Peer(str(service.host), sclass.server_features(env), 'env')
+                         for service in env.report_services]
+        self.server_version_args = sclass.server_version_args()
+        # Peers have one entry per hostname.  Once connected, the
+        # ip_addr property is either None, an onion peer, or the
+        # IP address that was connected to.  Adding a peer will evict
+        # any other peers with the same host name or IP address.
+        self.peers = set()
+        self.permit_onion_peer_time = time.time()
+        self.proxy = None
+        self.group = OldTaskGroup()
+        self.recent_peer_adds = {}
+        # refreshed
+        self.blacklist = set()
+
+    def _my_clearnet_peer(self):
+        '''Returns the clearnet peer representing this server, if any.'''
+        clearnet = [peer for peer in self.myselves if not peer.is_tor]
+        return clearnet[0] if clearnet else None
+
+    def _set_peer_statuses(self):
+        '''Set peer statuses.'''
+        cutoff = time.time() - STALE_SECS
+        for peer in self.peers:
+            if peer.bad:
+                peer.status = PEER_BAD
+            elif peer.last_good > cutoff:
+                peer.status = PEER_GOOD
+            elif peer.last_good:
+                peer.status = PEER_STALE
+            else:
+                peer.status = PEER_NEVER
+
+    def _features_to_register(self, peer, remote_peers):
+        '''If we should register ourselves to the remote peer, which has
+        reported the given list of known peers, return the clearnet
+        identity features to register, otherwise None.
+        '''
+        # Announce ourself if not present.  Don't if disabled, we
+        # are a non-public IP address, or to ourselves.
+        if not self.env.peer_announce or peer in self.myselves:
+            return None
+        my = self._my_clearnet_peer()
+        if not my or not my.is_public:
+            return None
+        # Register if no matches, or ports have changed
+        for peer in my.matches(remote_peers):
+            if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port:
+                return None
+        return my.features
+
+    def _permit_new_onion_peer(self, now):
+        '''Accept a new onion peer only once per random time interval.'''
+        if now < self.permit_onion_peer_time:
+            return False
+        self.permit_onion_peer_time = now + random.randrange(0, 1200)
+        return True
+
+    async def _import_peers(self):
+        '''Import hard-coded peers from a file or the coin defaults.'''
+        imported_peers = self.myselves.copy()
+        # Add the hard-coded ones unless only reporting ourself
+        if self.env.peer_discovery != self.env.PD_SELF:
+            imported_peers.extend(Peer.from_real_name(real_name, 'coins.py')
+                                  for real_name in self.env.coin.PEERS)
+        await self._note_peers(imported_peers, limit=None)
+
+    async def _refresh_blacklist(self):
+        url = self.env.blacklist_url
+        if not url:
+            return
+
+        async def read_blacklist():
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as response:
+                    text = await response.text()
+            return {entry.lower() for entry in json_deserialize(text)}
+
+        while True:
+            try:
+                self.blacklist = await read_blacklist()
+            except Exception as e:
+                self.logger.error(f'could not retrieve blacklist from {url}: {e}')
+            else:
+                self.logger.info(f'blacklist from {url} has {len(self.blacklist)} entries')
+                # Got new blacklist. Now check our current peers against it
+                for peer in self.peers:
+                    if self._is_blacklisted(peer):
+                        peer.retry_event.set()
+            await sleep(600)
+
+    def _is_blacklisted(self, peer):
+        host = peer.host.lower()
+        second_level_domain = '*.' + '.'.join(host.split('.')[-2:])
+        return any(item in self.blacklist
+                   for item in (host, second_level_domain, peer.ip_addr))
+
+    def _get_recent_good_peers(self):
+        cutoff = time.time() - STALE_SECS
+        recent = [peer for peer in self.peers
+                  if peer.last_good > cutoff and
+                  not peer.bad and peer.is_public]
+        recent = [peer for peer in recent if not self._is_blacklisted(peer)]
+        return recent
+
+    async def _detect_proxy(self):
+        '''Detect a proxy if we don't have one and some time has passed since
+        the last attempt.
+
+        If found self.proxy is set to a SOCKSProxy instance, otherwise None.
+        '''
+        host = self.env.tor_proxy_host
+        if self.env.tor_proxy_port is None:
+            ports = [9050, 9150, 1080]
+        else:
+            ports = [self.env.tor_proxy_port]
+        while True:
+            self.logger.info(f'trying to detect proxy on "{host}" '
+                             f'ports {ports}')
+            proxy = await SOCKSProxy.auto_detect_at_host(host, ports, None)
+            if proxy:
+                self.proxy = proxy
+                self.logger.info(f'detected {proxy}')
+                return
+            self.logger.info('no proxy detected, will try later')
+            await sleep(900)
+
+    async def _note_peers(self, peers, limit=2, check_ports=False, source=None):
+        '''Add a limited number of peers that are not already present.'''
+        new_peers = []
+        match_set = self.peers.copy()
+        for peer in peers:
+            if not peer.is_public or (peer.is_tor and not self.proxy):
+                continue
+
+            matches = peer.matches(match_set)
+            if matches:
+                if check_ports:
+                    for match in matches:
+                        if match.check_ports(peer):
+                            self.logger.info(f'ports changed for {peer}')
+                            # Retry connecting to the peer. First we will try the existing
+                            # ports and then try the new ports. Note that check_ports above
+                            # had a side_effect to temporarily store the new ports.
+                            # If we manage to connect, we will call 'server.features',
+                            # and the ports for this peer will be updated to the return values.
+                            match.retry_event.set()
+            else:
+                match_set.add(peer)
+                new_peers.append(peer)
+
+        if new_peers:
+            source = source or new_peers[0].source
+            if limit:
+                random.shuffle(new_peers)
+                use_peers = new_peers[:limit]
+            else:
+                use_peers = new_peers
+            for peer in use_peers:
+                self.logger.info(f'accepted new peer {peer} from {source}')
+                peer.retry_event = Event()
+                self.peers.add(peer)
+                await self.group.spawn(self._monitor_peer(peer))
+
+        return True
+
+    async def _monitor_peer(self, peer):
+        # Stop monitoring if we were dropped (a duplicate peer)
+        while peer in self.peers:
+            if await self._should_drop_peer(peer):
+                self.peers.discard(peer)
+                break
+            # Figure out how long to sleep before retrying.  Retry a
+            # good connection when it is about to turn stale, otherwise
+            # exponentially back off retries.
+            if peer.try_count == 0:
+                pause = STALE_SECS - WAKEUP_SECS * 2
+            else:
+                pause = WAKEUP_SECS * 2 ** peer.try_count
+            async with ignore_after(pause):
+                await peer.retry_event.wait()
+                peer.retry_event.clear()
+
+    async def _should_drop_peer(self, peer):
+        peer.try_count += 1
+        is_good = False
+        for kind, port, family in peer.connection_tuples():
+            peer.last_try = time.time()
+
+            kwargs = {'family': family}
+            if kind == 'SSL':
+                sslc = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+                sslc.check_hostname = False
+                sslc.verify_mode = ssl.CERT_NONE
+                kwargs['ssl'] = sslc
+
+            if self.env.force_proxy or peer.is_tor:
+                if not self.proxy:
+                    return
+                kwargs['proxy'] = self.proxy
+                kwargs['resolve'] = not peer.is_tor
+            else:
+                # Use our listening Host/IP for outgoing non-proxy
+                # connections so our peers see the correct source.
+                local_hosts = {service.host for service in self.env.services
+                               if isinstance(service.host, (IPv4Address, IPv6Address))
+                               and service.protocol != 'rpc'}
+                if local_hosts:
+                    kwargs['local_addr'] = (str(local_hosts.pop()), None)
+
+            session_factory = partial(
+                PeerSession,
+                manager_taskgroup=self.group,
+            )
+            peer_text = f'[{peer}:{port} {kind}]'
+            try:
+                async with connect_rs(
+                        peer.host, port,
+                        session_factory=session_factory,
+                        transport=PaddedRSTransport,
+                        **kwargs,
+                ) as session:
+                    session.sent_request_timeout = 120 if peer.is_tor else 30
+                    await self._verify_peer(session, peer)
+                is_good = True
+                break
+            except BadPeerError as e:
+                self.logger.error(f'{peer_text} marking bad: ({e})')
+                peer.mark_bad()
+                break
+            except CodeMessageError as e:
+                self.logger.error(f'{peer_text} RPC error: {e.message} '
+                                  f'({e.code})')
+            except (OSError, SOCKSError, ConnectionError, TaskTimeout) as e:
+                self.logger.info(f'{peer_text} {e}')
+
+        if is_good:
+            # Monotonic time would be better, but last_good and last_try are
+            # exported to admin RPC client.
+            now = time.time()
+            elapsed = now - peer.last_try
+            self.logger.info(f'{peer_text} verified in {elapsed:.1f}s')
+            peer.try_count = 0
+            peer.last_good = now
+            peer.source = 'peer'
+            # At most 2 matches if we're a host name, potentially
+            # several if we're an IP address (several instances
+            # can share a NAT).
+            matches = peer.matches(self.peers)
+            for match in matches:
+                if match.ip_address:
+                    if len(matches) > 1:
+                        self.peers.remove(match)
+                        # Force the peer's monitoring task to exit
+                        match.retry_event.set()
+                elif peer.host in match.features['hosts']:
+                    match.update_features_from_peer(peer)
+            # Trim this data structure
+            self.recent_peer_adds = {k: v for k, v in self.recent_peer_adds.items()
+                                     if v + PEER_ADD_PAUSE < now}
+        else:
+            # Forget the peer if long-term unreachable
+            if peer.last_good and not peer.bad:
+                try_limit = 10
+            else:
+                try_limit = 3
+            if peer.try_count >= try_limit:
+                desc = 'bad' if peer.bad else 'unreachable'
+                self.logger.info(f'forgetting {desc} peer: {peer}')
+                return True
+        return False
+
+    async def _verify_peer(self, session, peer):
+        # store IP address for peer
+        if not peer.is_tor:
+            address = session.remote_address()
+            if address and isinstance(address.host, (IPv4Address, IPv6Address)):
+                peer.ip_addr = str(address.host)
+
+        if self._is_blacklisted(peer):
+            raise BadPeerError('blacklisted')
+
+        # Bucket good recent peers; forbid many servers from similar IPs
+        # FIXME there's a race here, when verifying multiple peers
+        #       that belong to the same bucket ~simultaneously
+        recent_peers = self._get_recent_good_peers()
+        if peer in recent_peers:
+            recent_peers.remove(peer)
+        onion_peers = []
+        buckets = defaultdict(list)
+        for other_peer in recent_peers:
+            if other_peer.is_tor:
+                onion_peers.append(other_peer)
+            else:
+                buckets[other_peer.bucket_for_internal_purposes()].append(other_peer)
+        if peer.is_tor:
+            # keep number of onion peers below half of all peers,
+            # but up to 100 is OK regardless
+            if len(onion_peers) > len(recent_peers) // 2 >= 100:
+                raise BadPeerError('too many onion peers already')
+        else:
+            bucket = peer.bucket_for_internal_purposes()
+            if buckets[bucket]:
+                raise BadPeerError(f'too many peers already in bucket {bucket}')
+
+        # server.version goes first
+        message = 'server.version'
+        try:
+            result = await session.send_request(message, self.server_version_args)
+        except asyncio.CancelledError:
+            raise BadPeerError('terminated before server.version response')
+
+        assert_good(message, result, list)
+
+        # Protocol version 1.1 returns a pair with the version first
+        if len(result) != 2 or not all(isinstance(x, str) for x in result):
+            raise BadPeerError(f'bad server.version result: {result}')
+        server_version, _protocol_version = result
+        peer.server_version = server_version
+        peer.features['server_version'] = server_version
+
+        async with OldTaskGroup() as g:
+            await g.spawn(self._send_headers_subscribe(session))
+            await g.spawn(self._send_server_features(session, peer))
+            peers_task = await g.spawn(self._send_peers_subscribe
+                                       (session, peer))
+
+        # Process reported peers if remote peer is good
+        peers = peers_task.result()
+        await self._note_peers(peers)
+
+        features = self._features_to_register(peer, peers)
+        if features:
+            self.logger.info(f'registering ourself with {peer}')
+            # We only care to wait for the response
+            try:
+                await session.send_request('server.add_peer', [features])
+            except asyncio.CancelledError:
+                raise BadPeerError('terminated before server.add_peer response')
+
+    async def _send_headers_subscribe(self, session):
+        message = 'blockchain.headers.subscribe'
+        result = await session.send_request(message)
+        assert_good(message, result, dict)
+
+        # Compare heights.
+        MAX_TIP_HEIGHT_DIFF = 5
+        our_height = self.db.db_height
+        their_height = result.get('height')
+        if not isinstance(their_height, int):
+            raise BadPeerError(f'invalid height {their_height}')
+        if abs(our_height - their_height) > MAX_TIP_HEIGHT_DIFF:
+            raise BadPeerError(f'bad height {their_height:,d} '
+                               f'(ours: {our_height:,d})')
+
+        # Check prior header too in case of hard fork.
+        # Allow short chain-splits. On Bitcoin testnet4, <6 block splits and reorgs are *very* common.
+        MAX_CHAINSPLIT_LEN = 10
+        check_height = max(0, min(our_height, their_height) - MAX_CHAINSPLIT_LEN)
+        raw_header = await self.db.raw_header(check_height)
+        ours = raw_header.hex()
+        message = 'blockchain.block.header'
+        theirs = await session.send_request(message, [check_height])
+        assert_good(message, theirs, str)
+        if ours != theirs:
+            raise BadPeerError(f'our header {ours} and '
+                               f'theirs {theirs} differ')
+
+    async def _send_server_features(self, session, peer):
+        message = 'server.features'
+        features = await session.send_request(message)
+        assert_good(message, features, dict)
+        features_hosts = features.get('hosts', {})
+        assert_good(message, features_hosts, dict)
+        hosts = [host.lower() for host in features_hosts]
+        if self.env.coin.GENESIS_HASH != features.get('genesis_hash'):
+            raise BadPeerError('incorrect genesis hash')
+        if peer.host.lower() in hosts:
+            peer.update_features(features)
+        else:
+            raise BadPeerError(f'not listed in own hosts list {hosts}')
+
+    async def _send_peers_subscribe(self, session, peer):
+        message = 'server.peers.subscribe'
+        raw_peers = await session.send_request(message)
+        assert_good(message, raw_peers, list)
+
+        # Check the peers list we got from a remote peer.
+        # Each is expected to be of the form:
+        #    [ip_addr, hostname, ['v1.0', 't51001', 's51002']]
+        # Call add_peer if the remote doesn't appear to know about us.
+        try:
+            real_names = [' '.join([u[1]] + u[2]) for u in raw_peers]
+            return [Peer.from_real_name(real_name, str(peer))
+                    for real_name in real_names]
+        except Exception:
+            raise BadPeerError('bad server.peers.subscribe response')
+
+    #
+    # External interface
+    #
+    async def discover_peers(self):
+        '''Perform peer maintenance.  This includes
+
+          1) Forgetting unreachable peers.
+          2) Verifying connectivity of new peers.
+          3) Retrying old peers at regular intervals.
+        '''
+        self.logger.info(f'peer discovery: {self.env.peer_discovery}')
+        if self.env.peer_discovery != self.env.PD_ON:
+            self.logger.info('peer discovery is disabled')
+            return
+
+        self.logger.info(f'announce ourself: {self.env.peer_announce}')
+        self.logger.info(f'my clearnet self: {self._my_clearnet_peer()}')
+        self.logger.info(f'force use of proxy: {self.env.force_proxy}')
+        self.logger.info(f'beginning peer discovery...')
+        try:
+            async with self.group as group:
+                await group.spawn(self._refresh_blacklist())
+                await group.spawn(self._detect_proxy())
+                await group.spawn(self._import_peers())
+        except Exception:
+            self.logger.exception(
+                "PeerManager.group died! Please open a bug report about this, "
+                "and include at least this traceback and maybe a bit more of "
+                "the log (if it looks relevant).")
+            while True:
+                await sleep(3600)
+                self.logger.warning(
+                    "Peer discovery is not working as PeerManager.group died. "
+                    "This is not normal. You should inspect the log.")
+
+    def info(self):
+        '''The number of peers.'''
+        self._set_peer_statuses()
+        counter = Counter(peer.status for peer in self.peers)
+        return {
+            'bad': counter[PEER_BAD],
+            'good': counter[PEER_GOOD],
+            'never': counter[PEER_NEVER],
+            'stale': counter[PEER_STALE],
+            'total': len(self.peers),
+        }
+
+    async def add_localRPC_peer(self, real_name):
+        '''Add a peer passed by the admin over LocalRPC.'''
+        await self._note_peers([Peer.from_real_name(real_name, 'RPC')], check_ports=True)
+
+    async def on_add_peer(self, features: dict[str, Any] | Any, source_addr: Optional[str]):
+        '''Add a peer (but only if the peer resolves to the source).'''
+        if self.env.peer_discovery != self.env.PD_ON:
+            return False
+        if not source_addr:
+            self.logger.info('ignored add_peer request: no source info')
+            return False
+        source = str(source_addr.host)
+        peers = Peer.peers_from_features(features, source)
+        if not peers:
+            self.logger.info('ignored add_peer request: no peers given')
+            return False
+
+        # Just look at the first peer, require it
+        peer = peers[0]
+        host = peer.host
+        now = time.time()
+
+        # Rate limit peer adds by domain to one every 10 minutes
+        if peer.ip_address is not None:
+            bucket = 'ip_addr'
+        else:
+            bucket = '.'.join(host.lower().split('.')[-2:])
+        last = self.recent_peer_adds.get(bucket, 0)
+        self.recent_peer_adds[bucket] = now
+        if last + PEER_ADD_PAUSE >= now:
+            return False
+
+        if peer.is_tor:
+            permit = self._permit_new_onion_peer(now)
+            reason = 'rate limiting'
+        else:
+            getaddrinfo = asyncio.get_event_loop().getaddrinfo
+            try:
+                infos = await getaddrinfo(host, 80, type=socket.SOCK_STREAM)
+            except socket.gaierror:
+                permit = False
+                reason = 'address resolution failure'
+            else:
+                permit = any(source == info[-1][0] for info in infos)
+                reason = 'source-destination mismatch'
+
+        if permit:
+            self.logger.info(f'accepted add_peer request from {source} for {host}')
+            await self._note_peers([peer], check_ports=True)
+        else:
+            self.logger.warning(f'rejected add_peer request from {source} '
+                                f'for {host} ({reason})')
+
+        return permit
+
+    def on_peers_subscribe(self, is_tor):
+        '''Returns the server peers as a list of (ip, host, details) tuples.
+
+        We return all peers we've connected to in the last day.
+        Additionally, if we don't have onion routing, we return a few
+        hard-coded onion servers.
+        '''
+        recent = self._get_recent_good_peers()
+
+        # Always report ourselves if valid (even if not public)
+        cutoff = time.time() - STALE_SECS
+        peers = {
+            myself
+            for myself in self.myselves
+            if myself.last_good > cutoff
+        }
+
+        # Bucket the clearnet peers and select up to two from each
+        onion_peers = []
+        buckets = defaultdict(list)
+        for peer in recent:
+            if peer.is_tor:
+                onion_peers.append(peer)
+            else:
+                buckets[peer.bucket_for_external_interface()].append(peer)
+        for bucket_peers in buckets.values():
+            random.shuffle(bucket_peers)
+            peers.update(bucket_peers[:2])
+
+        # Add up to 20% onion peers (but up to 10 is OK anyway)
+        random.shuffle(onion_peers)
+        max_onion = 50 if is_tor else max(10, len(peers) // 4)
+
+        peers.update(onion_peers[:max_onion])
+
+        return [peer.to_tuple() for peer in peers]
+
+    def proxy_address(self):
+        '''Return the NetAddress of the proxy, if there is a proxy, otherwise
+        None.'''
+        return self.proxy.address if self.proxy else None
+
+    def rpc_data(self):
+        '''Peer data for the peers RPC method.'''
+        self._set_peer_statuses()
+
+        def peer_data(peer):
+            data = peer.serialize()
+            data['status'] = STATUS_DESCS[peer.status]
+            return data
+
+        def peer_key(peer):
+            return peer.bad, -peer.last_good
+
+        return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)]

src/electrumx/server/storage.py

@@ -0,0 +1,219 @@
+# Copyright (c) 2016-2017, the ElectrumX authors
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Backend database abstraction.'''
+
+import os
+from functools import partial
+from typing import Type, Sequence, TYPE_CHECKING
+
+import electrumx.lib.util as util
+
+if TYPE_CHECKING:
+    from rocksdb import DB as rocksdb_db
+    from plyvel import DB as plyvel_db
+
+
+def db_class(name: str) -> Type['Storage']:
+    '''Returns a DB engine class.'''
+    for db_class in util.subclasses(Storage):
+        if db_class.__name__.lower() == name.lower():
+            db_class.import_module()
+            return db_class
+    raise RuntimeError(f'unrecognised DB engine "{name}"')
+
+
+def list_db_engine_choices() -> Sequence[str]:
+    return [db_class.__name__.lower() for db_class in util.subclasses(Storage)]
+
+
+class Storage:
+    '''Abstract base class of the DB backend abstraction.'''
+
+    def __init__(self, name: str, for_sync: bool):
+        self.is_new = not os.path.exists(name)
+        self.for_sync = for_sync or self.is_new
+        self.open(name, create=self.is_new)
+
+    @classmethod
+    def import_module(cls) -> None:
+        '''Import the DB engine module.'''
+        raise NotImplementedError
+
+    def open(self, name: str, create: bool):
+        '''Open an existing database or create a new one.'''
+        raise NotImplementedError
+
+    def close(self) -> None:
+        '''Close an existing database.'''
+        raise NotImplementedError
+
+    def get(self, key: bytes) -> bytes:
+        raise NotImplementedError
+
+    def put(self, key: bytes, value: bytes) -> None:
+        raise NotImplementedError
+
+    def write_batch(self):
+        '''Return a context manager that provides `put` and `delete`.
+
+        Changes should only be committed when the context manager
+        closes without an exception.
+        '''
+        raise NotImplementedError
+
+    def iterator(self, prefix: bytes = b'', reverse: bool = False):
+        '''Return an iterator that yields (key, value) pairs from the
+        database sorted by key.
+
+        If `prefix` is set, only keys starting with `prefix` will be
+        included.  If `reverse` is True the items are returned in
+        reverse order.
+
+        The iterator supports .seek(key), which moves it just left of `key`.
+        - if forward-iterating
+            - if `key` is present, it will be the next item
+            - if `key` is not present, the next item will be the smallest still greater than `key`
+        - if reverse-iterating
+            - the next item will be the largest still smaller than `key`
+        '''
+        raise NotImplementedError
+
+
+class LevelDB(Storage):
+    '''LevelDB database engine.'''
+
+    db: 'plyvel_db'
+
+    @classmethod
+    def import_module(cls):
+        try:
+            import plyvel
+        except ImportError as e:
+            raise Exception(f"{e}. For the leveldb DB-backend, you should install ElectrumX with the [leveldb] pip extra.") from e
+
+        cls.module = plyvel
+
+    def open(self, name, create):
+        mof = 512 if self.for_sync else 128
+        # Use snappy compression (the default)
+        self.db = self.module.DB(name, create_if_missing=create,
+                                 max_open_files=mof)
+        self.close = self.db.close
+        self.get = self.db.get
+        self.put = self.db.put
+        self.write_batch = partial(self.db.write_batch, transaction=True,
+                                   sync=True)
+
+    def iterator(self, prefix=b'', reverse=False):
+        return LevelDBIterator(db=self.db, prefix=prefix, reverse=reverse)
+
+
+class LevelDBIterator:
+    '''An iterator for LevelDB.'''
+
+    def __init__(self, *, db: 'plyvel_db', prefix: bytes, reverse: bool):
+        self.prefix = prefix
+        self.iterator = db.iterator(prefix=prefix, reverse=reverse)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        k, v = next(self.iterator)
+        if not k.startswith(self.prefix):
+            raise StopIteration
+        return k, v
+
+    def seek(self, key: bytes) -> None:
+        self.iterator.seek(key)
+
+
+class RocksDB(Storage):
+    '''RocksDB database engine.'''
+
+    db: 'rocksdb_db'
+
+    @classmethod
+    def import_module(cls):
+        try:
+            import rocksdb
+        except ImportError as e:
+            raise Exception(f"{e}. For the rocksdb DB-backend, you should install ElectrumX with the [rocksdb] pip extra.") from e
+        cls.module = rocksdb
+
+    def open(self, name, create):
+        mof = 512 if self.for_sync else 128
+        # Use snappy compression (the default)
+        options = self.module.Options(create_if_missing=create,
+                                      use_fsync=True,
+                                      target_file_size_base=33554432,
+                                      max_open_files=mof)
+        self.db = self.module.DB(name, options)
+        self.close = self.db.close
+        self.get = self.db.get
+        self.put = self.db.put
+
+    def write_batch(self):
+        return RocksDBWriteBatch(self.db)
+
+    def iterator(self, prefix=b'', reverse=False):
+        return RocksDBIterator(db=self.db, prefix=prefix, reverse=reverse)
+
+
+class RocksDBWriteBatch:
+    '''A write batch for RocksDB.'''
+
+    def __init__(self, db: 'rocksdb_db'):
+        self.batch = RocksDB.module.WriteBatch()
+        self.db = db
+
+    def __enter__(self):
+        return self.batch
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not exc_val:
+            self.db.write(self.batch)
+
+
+class RocksDBIterator:
+    '''An iterator for RocksDB.'''
+
+    def __init__(self, *, db: 'rocksdb_db', prefix: bytes, reverse: bool):
+        self.prefix = prefix
+        self._is_reverse = reverse
+        if reverse:
+            self.iterator = reversed(db.iteritems())
+            nxt_prefix = util.increment_byte_string(prefix)
+            if nxt_prefix:
+                self.iterator.seek(nxt_prefix)
+                try:
+                    next(self.iterator)
+                except StopIteration:
+                    self.iterator.seek(nxt_prefix)
+            else:
+                self.iterator.seek_to_last()
+        else:
+            self.iterator = db.iteritems()
+            self.iterator.seek(prefix)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        k, v = next(self.iterator)
+        if not k.startswith(self.prefix):
+            raise StopIteration
+        return k, v
+
+    def seek(self, key: bytes) -> None:
+        self.iterator.seek(key)
+        if self._is_reverse:
+            try:
+                next(self)
+            except StopIteration:
+                self.iterator.seek_to_last()

src/electrumx/server/transport.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2025, The Electrum developers
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+import asyncio
+import time
+from typing import Optional, TYPE_CHECKING
+
+from aiorpcx.rawsocket import RSTransport
+
+if TYPE_CHECKING:
+    from .session import RPCSessionWithTaskGroup
+
+
+class PaddedRSTransport(RSTransport):
+    """A raw socket transport that provides basic countermeasures against traffic analysis
+    by padding the jsonrpc payload with whitespaces to have ~uniform-size TCP packets.
+    (it is assumed that a network observer does not see plaintext transport contents,
+    due to it being wrapped e.g. in TLS)
+    """
+
+    MIN_PACKET_SIZE = 1024
+    WAIT_FOR_BUFFER_GROWTH_SECONDS = 1.0
+    # amount of (unpadded) bytes sent instantly before beginning with polling.
+    # This makes the initial handshake where a few small messages are exchanged faster.
+    WARMUP_BUDGET_SIZE = 1024
+
+    session: Optional['RPCSessionWithTaskGroup']
+
+    def __init__(self, *args, **kwargs):
+        RSTransport.__init__(self, *args, **kwargs)
+        self._sbuffer = bytearray()  # "send buffer"
+        self._sbuffer_task = None  # type: Optional[asyncio.Task]
+        self._sbuffer_has_data_evt = asyncio.Event()
+        self._last_send = time.monotonic()
+        self._force_send = False  # type: bool
+
+    # note: this does not call super().write() but is a complete reimplementation
+    async def write(self, message):
+        await self._can_send.wait()
+        if self.is_closing():
+            return
+        framed_message = self._framer.frame(message)
+        self._sbuffer += framed_message
+        self._sbuffer_has_data_evt.set()
+        self._maybe_consume_sbuffer()
+
+    def _maybe_consume_sbuffer(self) -> None:
+        """Maybe take some data from sbuffer and send it on the wire."""
+        if not self._can_send.is_set() or self.is_closing():
+            return
+        buf = self._sbuffer
+        if not buf:
+            return
+        # if there is enough data in the buffer, or if we haven't sent in a while, send now:
+        if not (
+            self._force_send
+            or len(buf) >= self.MIN_PACKET_SIZE
+            or self._last_send + self.WAIT_FOR_BUFFER_GROWTH_SECONDS < time.monotonic()
+            or self.session.send_size < self.WARMUP_BUDGET_SIZE
+        ):
+            return
+        assert buf[-2:] in (b"}\n", b"]\n"), f"unexpected json-rpc terminator: {buf[-2:]=!r}"
+        # either (1) pad length to next power of two, to create "lsize" packet:
+        payload_lsize = len(buf)
+        total_lsize = max(self.MIN_PACKET_SIZE, 2 ** (payload_lsize.bit_length()))
+        npad_lsize = total_lsize - payload_lsize
+        # or if that wasted a lot of bandwidth with padding, (2) defer sending some messages
+        # and create a packet with half that size ("ssize", s for small)
+        total_ssize = max(self.MIN_PACKET_SIZE, total_lsize // 2)
+        payload_ssize = buf.rfind(b"\n", 0, total_ssize)
+        if payload_ssize != -1:
+            payload_ssize += 1  # for "\n" char
+            npad_ssize = total_ssize - payload_ssize
+        else:
+            npad_ssize = float("inf")
+        # decide between (1) and (2):
+        if self._force_send or npad_lsize <= npad_ssize:
+            # (1) create "lsize" packet: consume full buffer
+            npad = npad_lsize
+            p_idx = payload_lsize
+        else:
+            # (2) create "ssize" packet: consume some, but defer some for later
+            npad = npad_ssize
+            p_idx = payload_ssize
+        # pad by adding spaces near end
+        json_rpc_terminator = buf[p_idx-2:p_idx]
+        assert json_rpc_terminator in (b"}\n", b"]\n"), f"unexpected {json_rpc_terminator=!r}"
+        buf2 = buf[:p_idx-2] + (npad * b" ") + json_rpc_terminator
+        self._asyncio_transport.write(buf2)
+        self._last_send = time.monotonic()
+        del self._sbuffer[:p_idx]
+        if not self._sbuffer:
+            self._sbuffer_has_data_evt.clear()
+
+    async def _poll_sbuffer(self):
+        while not self.is_closing():
+            await self._can_send.wait()
+            await self._sbuffer_has_data_evt.wait()  # to avoid busy-waiting
+            self._maybe_consume_sbuffer()
+            # If there is still data in the buffer, sleep until it would time out.
+            # note: If the transport is ~idle, when we wake up, we will send the current buf data,
+            #       but if busy, we might wake up to completely new buffer contents. Either is fine.
+            if len(self._sbuffer) > 0:
+                timeout_abs = self._last_send + self.WAIT_FOR_BUFFER_GROWTH_SECONDS
+                timeout_rel = max(0.0, timeout_abs - time.monotonic())
+                await asyncio.sleep(timeout_rel)
+
+    def connection_made(self, transport: asyncio.BaseTransport):
+        super().connection_made(transport)
+        coro = self.session.taskgroup.spawn(self._poll_sbuffer())
+        self._sbuffer_task = self.loop.create_task(coro)
+
+    async def close(self, *args, **kwargs):
+        '''Close the connection and return when closed.'''
+        # Flush buffer before disconnecting. This makes ReplyAndDisconnect work:
+        self._force_send = True
+        self._maybe_consume_sbuffer()
+        await super().close(*args, **kwargs)