Initial commit: snapshot of spesmilo/electrumx 2.0.0

Unmodified source tree of upstream commit 280cb339aa158e099eb936033eecd15719770651
("Merge pull request #389 from SomberNight/202607_release_2_0",
https://github.com/spesmilo/electrumx, release 2.0.0, 3 July 2026).

All Palladium-specific changes are in the commits that follow.

tests/server/test_api.py

@@ -0,0 +1,108 @@
+import asyncio
+from unittest import mock
+
+from aiorpcx import RPCError
+from electrumx import Controller, Env
+
+loop = asyncio.new_event_loop()
+
+
+def set_env():
+    env = mock.create_autospec(Env)
+    env.coin = mock.Mock()
+    env.coin.SESSIONCLS.protocol_min_max_strings = lambda : ["1.1", "1.4"]
+    env.loop_policy = None
+    env.max_sessions = 0
+    env.max_subs = 0
+    env.max_send = 0
+    env.bandwidth_limit = 0
+    env.identities = ''
+    env.tor_proxy_host = env.tor_proxy_port = None
+    env.peer_discovery = env.PD_SELF = False
+    env.daemon_url = 'http://localhost:8000/'
+    return env
+
+
+async def coro(res):
+    return res
+
+
+def raise_exception(msg):
+    raise RPCError(1, msg)
+
+
+def ensure_text_exception(test, exception):
+    res = err = None
+    try:
+        res = loop.run_until_complete(test)
+    except Exception as e:
+        err = e
+    assert isinstance(err, exception), (res, err)
+
+
+def test_dummy():
+    assert True
+
+def _test_transaction_get():
+    async def test_verbose_ignore_by_backend():
+        env = set_env()
+        sut = Controller(env)
+        sut.daemon_request = mock.Mock()
+        sut.daemon_request.return_value = coro('11'*32)
+        res = await sut.transaction_get('ff'*32, True)
+        assert res == '11'*32
+
+    async def test_verbose_ok():
+        env = set_env()
+        sut = Controller(env)
+        sut.daemon_request = mock.Mock()
+        response = {
+            "hex": "00"*32,
+            "blockhash": "ff"*32
+        }
+        sut.daemon_request.return_value = coro(response)
+        res = await sut.transaction_get('ff'*32, True)
+        assert res == response
+
+        response = {
+            "hex": "00"*32,
+            "blockhash": None
+        }
+        sut.daemon_request.return_value = coro(response)
+        res = await sut.transaction_get('ff'*32, True)
+        assert res == response
+
+    async def test_no_verbose():
+        env = set_env()
+        sut = Controller(env)
+        sut.daemon_request = mock.Mock()
+        response = 'cafebabe'*64
+        sut.daemon_request.return_value = coro(response)
+        res = await sut.transaction_get('ff'*32)
+        assert res == response
+
+    async def test_verbose_failure():
+        env = set_env()
+        sut = Controller(env)
+        sut.daemon_request = mock.Mock()
+        sut.daemon_request.return_value = coro(
+            raise_exception('some unhandled error'))
+        await sut.transaction_get('ff' * 32, True)
+
+    async def test_wrong_txhash():
+        env = set_env()
+        sut = Controller(env)
+        sut.daemon_request = mock.Mock()
+        await sut.transaction_get('cafe')
+        sut.daemon_request.assert_not_called()
+
+    loop.run_until_complete(asyncio.gather(
+        *[
+            test_verbose_ignore_by_backend(),
+            test_verbose_ok(),
+            test_no_verbose()
+        ]
+    ))
+
+    for error_test in [test_verbose_failure, test_wrong_txhash]:
+        ensure_text_exception(error_test(), RPCError)

tests/server/test_daemon.py

@@ -0,0 +1,498 @@
+import aiohttp
+import asyncio
+import json
+import logging
+
+import pytest
+
+from aiorpcx import JSONRPCv1, JSONRPCLoose, RPCError, ignore_after, Request
+from electrumx.lib.coins import BitcoinCash, CoinError, Bitzeny, Dash
+from electrumx.server.daemon import Daemon, FakeEstimateFeeDaemon
+
+
+coin = BitcoinCash
+
+# These should be full, canonical URLs
+urls = ['http://rpc_user:rpc_pass@127.0.0.1:8332/',
+        'http://rpc_user:rpc_pass@192.168.0.1:8332/']
+
+
+@pytest.fixture(params=[BitcoinCash, Bitzeny])
+def daemon(request):
+    coin = request.param
+    return coin.DAEMON(coin, ','.join(urls))
+
+
+@pytest.fixture(params=[Dash])
+def dash_daemon(request):
+    coin = request.param
+    return coin.DAEMON(coin, ','.join(urls))
+
+
+class ResponseBase:
+
+    def __init__(self, headers, status):
+        self.headers = headers
+        self.status = status
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        pass
+
+
+class JSONResponse(ResponseBase):
+
+    def __init__(self, result, msg_id, status=200):
+        super().__init__({'Content-Type': 'application/json'}, status)
+        self.result = result
+        self.msg_id = msg_id
+
+    async def json(self, loads=json.loads):
+        if isinstance(self.msg_id, int):
+            message = JSONRPCv1.response_message(self.result, self.msg_id)
+        else:
+            parts = [JSONRPCv1.response_message(item, msg_id)
+                     for item, msg_id in zip(self.result, self.msg_id)]
+            message = JSONRPCv1.batch_message_from_parts(parts)
+        return loads(message.decode())
+
+
+class HTMLResponse(ResponseBase):
+
+    def __init__(self, text, reason, status):
+        super().__init__({'Content-Type': 'text/html; charset=ISO-8859-1'},
+                         status)
+        self._text = text
+        self.reason = reason
+
+    async def text(self):
+        return self._text
+
+
+class ClientSessionGood:
+    '''Imitate aiohttp for testing purposes.'''
+
+    def __init__(self, *triples):
+        self.triples = triples  # each a (method, args, result)
+        self.count = 0
+        self.expected_url = urls[0]
+
+    def post(self, url, data=""):
+        assert url == self.expected_url
+        request, request_id = JSONRPCLoose.message_to_item(data.encode())
+        method, args, result = self.triples[self.count]
+        self.count += 1
+        if isinstance(request, Request):
+            assert request.method == method
+            assert request.args == args
+            return JSONResponse(result, request_id)
+        else:
+            batch = request
+            assert isinstance(batch, list)
+            request_ids = []
+            for payload, args in zip(batch, args):
+                assert payload['method'] == method
+                assert payload['params'] == args
+                request_ids.append(payload['id'])
+            return JSONResponse(result, request_ids)
+
+
+class ClientSessionBadAuth:
+
+    def post(self, url, data=""):
+         return HTMLResponse('', 'Unauthorized', 401)
+
+
+class ClientSessionWorkQueueFull(ClientSessionGood):
+
+    def post(self, url, data=""):
+        self.post = super().post
+        return HTMLResponse('Work queue depth exceeded',
+                            'Internal server error', 500)
+
+
+class ClientSessionPostError(ClientSessionGood):
+
+    def __init__(self, exception, *args):
+        super().__init__(*args)
+        self.exception = exception
+        self.n = 0
+
+    def post(self, url, data=""):
+        self.n += 1
+        if self.n == 1:
+            raise self.exception
+        return super().post(url, data)
+
+
+class ClientSessionFailover(ClientSessionGood):
+
+    def post(self, url, data=""):
+        # If not failed over; simulate disconnecting
+        if url == self.expected_url:
+            raise aiohttp.ServerDisconnectedError
+        else:
+            self.expected_url = urls[1]
+            return super().post(url, data)
+
+
+def in_caplog(caplog, message, count=1):
+    return sum(message in record.message
+               for record in caplog.records) == count
+
+#
+# Tests
+#
+
+@pytest.mark.asyncio
+async def test_set_urls_bad():
+    with pytest.raises(CoinError):
+        Daemon(coin, '')
+    with pytest.raises(CoinError):
+        Daemon(coin, 'a')
+
+
+@pytest.mark.asyncio
+async def test_set_urls_one(caplog):
+    with caplog.at_level(logging.INFO):
+        daemon = Daemon(coin, urls[0])
+        assert daemon.current_url() == urls[0]
+        assert len(daemon.urls) == 1
+        logged_url = daemon.logged_url()
+        assert logged_url == '127.0.0.1:8332/'
+        assert in_caplog(caplog, f'daemon #1 at {logged_url} (current)')
+
+
+@pytest.mark.asyncio
+async def test_set_urls_two(caplog):
+    with caplog.at_level(logging.INFO):
+        daemon = Daemon(coin, ','.join(urls))
+        assert daemon.current_url() == urls[0]
+        assert len(daemon.urls) == 2
+        logged_url = daemon.logged_url()
+        assert logged_url == '127.0.0.1:8332/'
+        assert in_caplog(caplog, f'daemon #1 at {logged_url} (current)')
+        assert in_caplog(caplog, 'daemon #2 at 192.168.0.1:8332')
+
+
+@pytest.mark.asyncio
+async def test_set_urls_short():
+    no_prefix_urls = ['/'.join(part for part in url.split('/')[2:])
+                      for url in urls]
+    daemon = Daemon(coin, ','.join(no_prefix_urls))
+    assert daemon.current_url() == urls[0]
+    assert len(daemon.urls) == 2
+
+    no_slash_urls = [url[:-1] for url in urls]
+    daemon = Daemon(coin, ','.join(no_slash_urls))
+    assert daemon.current_url() == urls[0]
+    assert len(daemon.urls) == 2
+
+    no_port_urls = [url[:url.rfind(':')] for url in urls]
+    daemon = Daemon(coin, ','.join(no_port_urls))
+    assert daemon.current_url() == urls[0]
+    assert len(daemon.urls) == 2
+
+
+@pytest.mark.asyncio
+async def test_failover_good(caplog):
+    daemon = Daemon(coin, ','.join(urls))
+    with caplog.at_level(logging.INFO):
+        result = daemon.failover()
+    assert result is True
+    assert daemon.current_url() == urls[1]
+    logged_url = daemon.logged_url()
+    assert in_caplog(caplog, f'failing over to {logged_url}')
+    # And again
+    result = daemon.failover()
+    assert result is True
+    assert daemon.current_url() == urls[0]
+
+
+@pytest.mark.asyncio
+async def test_failover_fail(caplog):
+    daemon = Daemon(coin, urls[0])
+    with caplog.at_level(logging.INFO):
+        result = daemon.failover()
+    assert result is False
+    assert daemon.current_url() == urls[0]
+    assert not in_caplog(caplog, f'failing over')
+
+
+@pytest.mark.asyncio
+async def test_height(daemon):
+    assert daemon.cached_height() is None
+    height = 300
+    daemon.session = ClientSessionGood(('getblockcount', [], height))
+    assert await daemon.height() == height
+    assert daemon.cached_height() == height
+
+
+@pytest.mark.asyncio
+async def test_broadcast_transaction(daemon):
+    raw_tx = 'deadbeef'
+    tx_hash = 'hash'
+    daemon.session = ClientSessionGood(('sendrawtransaction', [raw_tx], tx_hash))
+    assert await daemon.broadcast_transaction(raw_tx) == tx_hash
+
+@pytest.mark.asyncio
+async def test_broadcast_package(daemon):
+    package = ["deadbeef", "deadc0de", "facefeed"]
+    result = {"package_msg": "success"}
+    daemon.session = ClientSessionGood(('submitpackage', [package], result))
+    assert await daemon.broadcast_package(package) == result
+
+@pytest.mark.asyncio
+async def test_relayfee(daemon):
+    if isinstance(daemon, FakeEstimateFeeDaemon):
+        sats = daemon.coin.ESTIMATE_FEE
+    else:
+        sats = 2
+    response = {"relayfee": sats, "other:": "cruft"}
+    daemon.session = ClientSessionGood(('getnetworkinfo', [], response))
+    assert await daemon.relayfee() == sats
+
+
+@pytest.mark.asyncio
+async def test_mempool_info(daemon):
+    bitcoin_per_kvb = 0.00002123
+    response = {
+        'mempoolminfee': bitcoin_per_kvb,
+        'minrelaytxfee': bitcoin_per_kvb,
+        'incrementalrelayfee': bitcoin_per_kvb,
+        'other': 'cruft',
+    }
+    daemon.session = ClientSessionGood(('getmempoolinfo', [], response))
+    expected_result = dict(response)
+    del expected_result['other']
+    assert await daemon.mempool_info() == expected_result
+
+
+@pytest.mark.asyncio
+async def test_mempool_txids_hum(daemon):
+    hashes = ['hex_hash1', 'hex_hash2']
+    daemon.session = ClientSessionGood(('getrawmempool', [], hashes))
+    assert await daemon.mempool_txids_hum() == hashes
+
+
+@pytest.mark.asyncio
+async def test_deserialised_block(daemon):
+    block_hash = 'block_hash'
+    result = {'some': 'mess'}
+    daemon.session = ClientSessionGood(('getblock', [block_hash, True], result))
+    assert await daemon.deserialised_block(block_hash) == result
+
+
+@pytest.mark.asyncio
+async def test_estimatefee(daemon):
+    method_not_found = RPCError(JSONRPCv1.METHOD_NOT_FOUND, 'nope')
+    if isinstance(daemon, FakeEstimateFeeDaemon):
+        result = daemon.coin.ESTIMATE_FEE
+    else:
+        result = -1
+    daemon.session = ClientSessionGood(
+            ('estimatesmartfee', [], method_not_found),
+            ('estimatefee', [2], result)
+    )
+    assert await daemon.estimatefee(2) == result
+
+
+@pytest.mark.asyncio
+async def test_estimatefee_smart(daemon):
+    bad_args = RPCError(JSONRPCv1.INVALID_ARGS, 'bad args')
+    if isinstance(daemon, FakeEstimateFeeDaemon):
+        return
+    rate = 0.0002
+    result = {'feerate': rate}
+    daemon.session = ClientSessionGood(
+        ('estimatesmartfee', [], bad_args),
+        ('estimatesmartfee', [2], result)
+    )
+    assert await daemon.estimatefee(2) == rate
+
+    # Test the rpc_available_cache is used
+    daemon.session = ClientSessionGood(('estimatesmartfee', [2], result))
+    assert await daemon.estimatefee(2) == rate
+
+
+@pytest.mark.asyncio
+async def test_getrawtransaction(daemon):
+    hex_hash = 'deadbeef'
+    simple = 'tx_in_hex'
+    verbose = {'hex': hex_hash, 'other': 'cruft'}
+    # Test False is converted to 0 - old daemon's reject False
+    daemon.session = ClientSessionGood(('getrawtransaction', [hex_hash, 0], simple))
+    assert await daemon.getrawtransaction(hex_hash) == simple
+
+    # Test True is converted to 1
+    daemon.session = ClientSessionGood(('getrawtransaction', [hex_hash, 1], verbose))
+    assert await daemon.getrawtransaction(
+        hex_hash, True) == verbose
+
+
+@pytest.mark.asyncio
+async def test_protx(dash_daemon):
+    protx_hash = 'deadbeaf'
+    dash_daemon.session = ClientSessionGood(('protx', ['info', protx_hash], {}))
+    assert await dash_daemon.protx(['info', protx_hash]) == {}
+
+
+# Batch tests
+
+@pytest.mark.asyncio
+async def test_empty_send(daemon):
+    first = 5
+    count = 0
+    daemon.session = ClientSessionGood(('getblockhash', [], []))
+    assert await daemon.block_hex_hashes(first, count) == []
+
+
+@pytest.mark.asyncio
+async def test_block_hex_hashes(daemon):
+    first = 5
+    count = 3
+    hashes = [f'hex_hash{n}' for n in range(count)]
+    daemon.session = ClientSessionGood(('getblockhash',
+                                        [[n] for n in range(first, first + count)],
+                                        hashes))
+    assert await daemon.block_hex_hashes(first, count) == hashes
+
+
+@pytest.mark.asyncio
+async def test_raw_blocks(daemon):
+    count = 3
+    hex_hashes = [f'hex_hash{n}' for n in range(count)]
+    args_list = [[hex_hash, False] for hex_hash in hex_hashes]
+    iterable = (hex_hash for hex_hash in hex_hashes)
+    blocks = ["00", "019a", "02fe"]
+    blocks_raw = [bytes.fromhex(block) for block in blocks]
+    daemon.session = ClientSessionGood(('getblock', args_list, blocks))
+    assert await daemon.raw_blocks(iterable) == blocks_raw
+
+
+@pytest.mark.asyncio
+async def test_get_raw_transactions(daemon):
+    hex_hashes = ['deadbeef0', 'deadbeef1']
+    args_list = [[hex_hash, 0] for hex_hash in hex_hashes]
+    raw_txs_hex = ['fffefdfc', '0a0b0c0d']
+    raw_txs = [bytes.fromhex(raw_tx) for raw_tx in raw_txs_hex]
+    # Test 0 - old daemon's reject False
+    daemon.session = ClientSessionGood(('getrawtransaction', args_list, raw_txs_hex))
+    assert await daemon.getrawtransactions(hex_hashes) == raw_txs
+
+    # Test one error
+    tx_not_found = RPCError(-1, 'some error message')
+    results = ['ff0b7d', tx_not_found]
+    raw_txs = [bytes.fromhex(results[0]), None]
+    daemon.session = ClientSessionGood(('getrawtransaction', args_list, results))
+    assert await daemon.getrawtransactions(hex_hashes) == raw_txs
+
+
+# Other tests
+
+@pytest.mark.asyncio
+async def test_bad_auth(daemon, caplog):
+    async with ignore_after(0.1):
+        daemon.session = ClientSessionBadAuth()
+        await daemon.height()
+
+    assert in_caplog(caplog, "daemon service refused")
+    assert in_caplog(caplog, "Unauthorized")
+
+
+@pytest.mark.asyncio
+async def test_workqueue_depth(daemon, caplog):
+    daemon.init_retry = 0.01
+    height = 125
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionWorkQueueFull(('getblockcount', [], height))
+        await daemon.height() == height
+
+    assert in_caplog(caplog, "Work queue depth exceeded")
+    assert in_caplog(caplog, "running normally")
+
+
+@pytest.mark.asyncio
+async def test_connection_error(daemon, caplog):
+    height = 100
+    daemon.init_retry = 0.01
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionPostError(aiohttp.ClientConnectionError,
+                                                ('getblockcount', [], height))
+        await daemon.height() == height
+
+    assert in_caplog(caplog, "connection problem - check your daemon is running")
+    assert in_caplog(caplog, "connection restored")
+
+
+@pytest.mark.asyncio
+async def test_timeout_error(daemon, caplog):
+    height = 100
+    daemon.init_retry = 0.01
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionPostError(asyncio.TimeoutError,
+                                                ('getblockcount', [], height))
+        await daemon.height() == height
+
+    assert in_caplog(caplog, "timeout error")
+
+
+@pytest.mark.asyncio
+async def test_disconnected(daemon, caplog):
+    height = 100
+    daemon.init_retry = 0.01
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionPostError(aiohttp.ServerDisconnectedError,
+                                                ('getblockcount', [], height))
+        await daemon.height() == height
+
+    assert in_caplog(caplog, "disconnected")
+    assert in_caplog(caplog, "connection restored")
+
+
+@pytest.mark.asyncio
+async def test_warming_up(daemon, caplog):
+    warming_up = RPCError(-28, 'reading block index')
+    height = 100
+    daemon.init_retry = 0.01
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionGood(
+            ('getblockcount', [], warming_up),
+            ('getblockcount', [], height)
+        )
+        assert await daemon.height() == height
+
+    assert in_caplog(caplog, "starting up checking blocks")
+    assert in_caplog(caplog, "running normally")
+
+
+@pytest.mark.asyncio
+async def test_warming_up_batch(daemon, caplog):
+    warming_up = RPCError(-28, 'reading block index')
+    first = 5
+    count = 1
+    daemon.init_retry = 0.01
+    hashes = ['hex_hash5']
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionGood(('getblockhash', [[first]], [warming_up]),
+                                           ('getblockhash', [[first]], hashes))
+        assert await daemon.block_hex_hashes(first, count) == hashes
+
+    assert in_caplog(caplog, "starting up checking blocks")
+    assert in_caplog(caplog, "running normally")
+
+
+@pytest.mark.asyncio
+async def test_failover(daemon, caplog):
+    height = 100
+    daemon.init_retry = 0.01
+    daemon.max_retry = 0.04
+    with caplog.at_level(logging.INFO):
+        daemon.session = ClientSessionFailover(('getblockcount', [], height))
+        await daemon.height() == height
+
+    assert in_caplog(caplog, "disconnected", 1)
+    assert in_caplog(caplog, "failing over")
+    assert in_caplog(caplog, "connection restored")

tests/server/test_env.py

@@ -0,0 +1,430 @@
+# Tests of server/env.py
+
+import os
+import random
+import re
+
+import pytest
+
+from aiorpcx import Service, NetAddress
+from electrumx.server.env import Env, ServiceError
+import electrumx.lib.coins as lib_coins
+
+
+BASE_DAEMON_URL = 'http://username:password@hostname:321/'
+BASE_DB_DIR = '/some/dir'
+
+base_environ = {
+    'DB_DIRECTORY': BASE_DB_DIR,
+    'DB_ENGINE': 'rocksdb',
+    'DAEMON_URL': BASE_DAEMON_URL,
+}
+
+
+def setup_base_env():
+    os.environ.clear()
+    os.environ.update(base_environ)
+
+
+def assert_required(env_var):
+    setup_base_env()
+    os.environ.pop(env_var, None)
+    with pytest.raises(Env.Error):
+        Env()
+
+
+def assert_default(env_var, attr, default):
+    setup_base_env()
+    e = Env()
+    assert getattr(e, attr) == default
+    os.environ[env_var] = 'foo'
+    e = Env()
+    assert getattr(e, attr) == 'foo'
+
+
+def assert_integer(
+        env_var, attr, default='', *, min_value=5, max_value=2000):
+    setup_base_env()
+    if default != '':
+        e = Env()
+        assert getattr(e, attr) == default
+    value = random.randrange(min_value, max_value)
+    os.environ[env_var] = str(value) + '.1'
+    with pytest.raises(Env.Error):
+        Env()
+    os.environ[env_var] = str(value)
+    e = Env()
+    assert getattr(e, attr) == value
+
+
+def assert_boolean(env_var, attr, default):
+    e = Env()
+    assert getattr(e, attr) == default
+    os.environ[env_var] = 'foo'
+    e = Env()
+    assert getattr(e, attr) is True
+    os.environ[env_var] = ''
+    e = Env()
+    assert getattr(e, attr) is False
+
+
+def test_minimal():
+    setup_base_env()
+    Env()
+
+
+def test_DB_DIRECTORY():
+    assert_required('DB_DIRECTORY')
+    setup_base_env()
+    e = Env()
+    assert e.db_dir == BASE_DB_DIR
+
+
+def test_DAEMON_URL():
+    assert_required('DAEMON_URL')
+    setup_base_env()
+    e = Env()
+    assert e.daemon_url == BASE_DAEMON_URL
+
+
+def test_COIN_NET():
+    '''Test COIN and NET defaults and redirection.'''
+    setup_base_env()
+    e = Env()
+    assert e.coin == lib_coins.BitcoinMainnet
+    os.environ['NET'] = 'testnet'
+    e = Env()
+    assert e.coin == lib_coins.BitcoinTestnet3
+    os.environ['NET'] = ' testnet '
+    e = Env()
+    assert e.coin == lib_coins.BitcoinTestnet3
+    os.environ.pop('NET')
+    os.environ['COIN'] = ' Litecoin '
+    e = Env()
+    assert e.coin == lib_coins.Litecoin
+    os.environ['NET'] = 'testnet'
+    e = Env()
+    assert e.coin == lib_coins.LitecoinTestnet
+    os.environ.pop('NET')
+    os.environ['COIN'] = ' BitcoinGold '
+    e = Env()
+    assert e.coin == lib_coins.BitcoinGold
+    os.environ['NET'] = 'testnet'
+    e = Env()
+    assert e.coin == lib_coins.BitcoinGoldTestnet
+    os.environ['NET'] = 'regtest'
+    e = Env()
+    assert e.coin == lib_coins.BitcoinGoldRegtest
+    os.environ.pop('NET')
+    os.environ['COIN'] = ' Decred '
+    e = Env()
+    assert e.coin == lib_coins.Decred
+    os.environ['NET'] = 'testnet'
+    e = Env()
+    assert e.coin == lib_coins.DecredTestnet
+    os.environ.pop('NET')
+    os.environ['COIN'] = ' BitcoinGreen '
+    e = Env()
+    assert e.coin == lib_coins.Bitg
+    os.environ['NET'] = 'mainnet'
+    e = Env()
+    os.environ.pop('NET')
+    os.environ['COIN'] = ' Pivx '
+    os.environ['NET'] = 'mainnet'
+    e = Env()
+    assert e.coin == lib_coins.Pivx
+    os.environ['NET'] = 'testnet'
+    e = Env()
+    assert e.coin == lib_coins.PivxTestnet
+    os.environ.pop('NET')
+    os.environ['NET'] = 'mainnet'
+    os.environ['COIN'] = ' TokenPay '
+    e = Env()
+    assert e.coin == lib_coins.TokenPay
+
+
+def test_CACHE_MB():
+    assert_integer('CACHE_MB', 'cache_MB', 1200)
+
+
+def test_SERVICES():
+    setup_base_env()
+    e = Env()
+    assert e.services == []
+    # This has a blank entry between commas
+    os.environ['SERVICES'] = 'tcp://foo.bar:1234,,ws://1.2.3.4:567,rpc://[::1]:700'
+    e = Env()
+    assert e.services == [
+        Service('tcp', NetAddress('foo.bar', 1234)),
+        Service('ws', NetAddress('1.2.3.4', 567)),
+        Service('rpc', NetAddress('::1', 700)),
+    ]
+
+
+def test_SERVICES_default_rpc():
+    # This has a blank entry between commas
+    os.environ['SERVICES'] = 'rpc://foo.bar'
+    e = Env()
+    assert e.services[0].host == 'foo.bar'
+    assert e.services[0].port == 8000
+    os.environ['SERVICES'] = 'rpc://:800'
+    e = Env()
+    assert e.services[0].host == 'localhost'
+    assert e.services[0].port == 800
+    os.environ['SERVICES'] = 'rpc://'
+    e = Env()
+    assert e.services[0].host == 'localhost'
+    assert e.services[0].port == 8000
+
+
+def test_bad_SERVICES():
+    setup_base_env()
+    os.environ['SERVICES'] = 'tcp:foo.bar:1234'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'invalid service string' in str(err.value)
+    os.environ['SERVICES'] = 'xxx://foo.com:50001'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'unknown protocol' in str(err.value)
+
+
+def test_onion_SERVICES():
+    setup_base_env()
+    os.environ['SERVICES'] = 'tcp://foo.bar.onion:1234'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'bad host' in str(err.value)
+
+
+def test_duplicate_SERVICES():
+    setup_base_env()
+    os.environ['SERVICES'] = 'tcp://foo.bar:1234,ws://foo.bar:1235'
+    Env()
+    os.environ['SERVICES'] = 'tcp://foo.bar:1234,ws://foo.bar:1234'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'multiple services' in str(err.value)
+
+
+@pytest.mark.parametrize("service", (
+    'ssl://foo.bar:1234',
+    'wss://foo.bar:1234',
+))
+def test_ssl_SERVICES(service):
+    setup_base_env()
+    os.environ['SERVICES'] = service
+    with pytest.raises(Env.Error) as err:
+        Env()
+    assert 'SSL_CERTFILE' in str(err.value)
+    os.environ['SSL_CERTFILE'] = 'certfile'
+    with pytest.raises(Env.Error) as err:
+        Env()
+    assert 'SSL_KEYFILE' in str(err.value)
+    os.environ['SSL_KEYFILE'] = 'keyfile'
+    Env()
+    setup_base_env()
+    os.environ['SERVICES'] = service
+    os.environ['SSL_KEYFILE'] = 'keyfile'
+    with pytest.raises(Env.Error) as err:
+        Env()
+    assert 'SSL_CERTFILE' in str(err.value)
+
+
+def test_REPORT_SERVICES():
+    setup_base_env()
+    e = Env()
+    assert e.report_services == []
+    # This has a blank entry between commas
+    os.environ['REPORT_SERVICES'] = 'tcp://foo.bar:1234,,ws://1.2.3.4:567'
+    e = Env()
+    assert e.report_services == [
+        Service('tcp', NetAddress('foo.bar', 1234)),
+        Service('ws', NetAddress('1.2.3.4', 567)),
+    ]
+
+
+def test_REPORT_SERVICES_rpc():
+    setup_base_env()
+    os.environ['REPORT_SERVICES'] = 'rpc://foo.bar:1234'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'bad protocol' in str(err.value)
+
+
+def test_REPORT_SERVICES_private():
+    setup_base_env()
+    os.environ['REPORT_SERVICES'] = 'tcp://192.168.0.1:1234'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'bad IP address' in str(err.value)
+    # Accept it not PEER_ANNOUNCE
+    os.environ['PEER_ANNOUNCE'] = ''
+    Env()
+
+
+def test_REPORT_SERVICES_localhost():
+    setup_base_env()
+    os.environ['REPORT_SERVICES'] = 'tcp://localhost:1234'
+    with pytest.raises(ServiceError) as err:
+        Env()
+    assert 'bad host' in str(err.value)
+
+
+def test_REORG_LIMIT():
+    assert_integer('REORG_LIMIT', 'reorg_limit',
+                   lib_coins.Bitcoin.REORG_LIMIT)
+
+
+def test_COST_HARD_LIMIT():
+    assert_integer(
+        'COST_HARD_LIMIT',
+        'cost_hard_limit',
+        10000,
+        min_value=5000,
+        max_value=20000,
+    )
+
+
+def test_COST_SOFT_LIMIT():
+    assert_integer('COST_SOFT_LIMIT', 'cost_soft_limit', 1000)
+
+
+def test_INITIAL_CONCURRENT():
+    assert_integer('INITIAL_CONCURRENT', 'initial_concurrent', 10)
+
+
+def test_REQUEST_SLEEP():
+    assert_integer('REQUEST_SLEEP', 'request_sleep', 2500)
+
+
+def test_BANDWIDTH_UNIT_COST():
+    assert_integer('BANDWIDTH_UNIT_COST', 'bw_unit_cost', 5000)
+
+
+def test_DONATION_ADDRESS():
+    assert_default('DONATION_ADDRESS', 'donation_address', '')
+
+
+def test_DB_ENGINE():
+    assert_required('DB_ENGINE')
+    setup_base_env()
+
+    os.environ['DB_ENGINE'] = "rocksdb"
+    e = Env()
+    assert e.db_engine == "rocksdb"
+
+    os.environ['DB_ENGINE'] = "leveldb"
+    e = Env()
+    assert e.db_engine == "leveldb"
+
+    os.environ['DB_ENGINE'] = "custom-something"
+    with pytest.raises(Env.Error):
+        e = Env()
+
+
+def test_MAX_SEND():
+    assert_integer('MAX_SEND', 'max_send', 8_100_000)
+
+
+def test_LOG_LEVEL():
+    setup_base_env()
+    e = Env()
+    assert e.log_level == 'INFO'
+    os.environ['LOG_LEVEL'] = 'warning'
+    e = Env()
+    assert e.log_level == 'WARNING'
+
+
+def test_MAX_SESSIONS():
+    too_big = 10_000_000_000
+    os.environ['MAX_SESSIONS'] = str(too_big)
+    e = Env()
+    assert e.max_sessions < too_big
+    # Cannot test default as it may be lowered by the open file limit cap
+
+
+def test_REQUEST_TIMEOUT():
+    assert_integer('REQUEST_TIMEOUT', 'request_timeout', 30)
+
+
+def test_SESSION_TIMEOUT():
+    assert_integer('SESSION_TIMEOUT', 'session_timeout', 600)
+
+
+def test_BANNER_FILE():
+    e = Env()
+    assert e.banner_file is None
+    assert e.tor_banner_file is None
+    os.environ['BANNER_FILE'] = 'banner_file'
+    e = Env()
+    assert e.banner_file == 'banner_file'
+    assert e.tor_banner_file == 'banner_file'
+    os.environ['TOR_BANNER_FILE'] = 'tor_banner_file'
+    e = Env()
+    assert e.banner_file == 'banner_file'
+    assert e.tor_banner_file == 'tor_banner_file'
+
+
+def test_EVENT_LOOP_POLICY():
+    e = Env()
+    assert e.loop_policy is None
+    os.environ['EVENT_LOOP_POLICY'] = 'foo'
+    with pytest.raises(Env.Error):
+        Env()
+    os.environ['EVENT_LOOP_POLICY'] = 'uvloop'
+    try:
+        Env()
+    except ImportError:
+        pass
+    del os.environ['EVENT_LOOP_POLICY']
+
+
+def test_ANON_LOGS():
+    assert_boolean('ANON_LOGS', 'anon_logs', False)
+
+
+def test_PEER_DISCOVERY():
+    e = Env()
+    assert e.peer_discovery == Env.PD_ON
+    os.environ['PEER_DISCOVERY'] = ' '
+    e = Env()
+    assert e.peer_discovery == Env.PD_OFF
+    os.environ['PEER_DISCOVERY'] = 'ON'
+    e = Env()
+    assert e.peer_discovery == Env.PD_ON
+    os.environ['PEER_DISCOVERY'] = 'self'
+    e = Env()
+    assert e.peer_discovery == Env.PD_SELF
+
+
+def test_PEER_ANNOUNCE():
+    assert_boolean('PEER_ANNOUNCE', 'peer_announce', True)
+
+
+def test_FORCE_PROXY():
+    assert_boolean('FORCE_PROXY', 'force_proxy', False)
+
+
+def test_TOR_PROXY_HOST():
+    assert_default('TOR_PROXY_HOST', 'tor_proxy_host', 'localhost')
+
+
+def test_TOR_PROXY_PORT():
+    assert_integer('TOR_PROXY_PORT', 'tor_proxy_port', None)
+
+
+def test_ban_versions():
+    e = Env()
+    assert e.drop_client is None
+    ban_re = r'1\.[0-2]\.\d+?[_\w]*'
+    os.environ['DROP_CLIENT'] = ban_re
+    e = Env()
+    assert e.drop_client == re.compile(ban_re)
+    assert e.drop_client.match("1.2.3_buggy_client")
+    assert e.drop_client.match("1.3.0_good_client") is None
+
+
+def test_coin_class_provided():
+    e = Env(lib_coins.Bitcoin)
+    assert e.coin == lib_coins.Bitcoin

tests/server/test_mempool.py

@@ -0,0 +1,606 @@
+import asyncio
+import dataclasses
+import datetime
+import logging
+import os
+from collections import defaultdict
+from functools import partial
+from random import randrange, choice, seed
+from typing import Iterable, Sequence
+
+import pytest
+from aiorpcx import Event, sleep, ignore_after
+
+from electrumx.server.mempool import MemPool, MemPoolAPI
+from electrumx.lib.coins import BitcoinCash
+from electrumx.lib.hash import HASHX_LEN, hex_str_to_hash, hash_to_hex_str
+from electrumx.lib.tx import Tx, TxInput, TxOutput
+from electrumx.lib.util import OldTaskGroup
+
+
+coin = BitcoinCash
+tx_hash_fn = coin.DESERIALIZER.TX_HASH_FN
+# Change seed daily
+seed(datetime.date.today().toordinal())
+
+
+def random_tx(hash160s, utxos):
+    '''Create a random TX paying to some of the hash160s using some of the
+    UTXOS.  Return the TX.  UTXOs is updated for the effects of the TX.
+    '''
+    inputs = []
+    n_inputs = min(randrange(1, 4), len(utxos))
+    input_value = 0
+    # Create inputs spending random UTXOs.  total the inpu
+    for n in range(n_inputs):
+        prevout = choice(list(utxos))
+        hashX, value = utxos.pop(prevout)
+        inputs.append(TxInput(
+            prev_txid_rev=prevout[0],
+            prev_idx=prevout[1],
+            script=b'',
+            sequence=4294967295,
+        ))
+        input_value += value
+
+    # Sometimes add a generation/coinbase like input that is present
+    # in some coins
+    if randrange(0, 10) == 0:
+        inputs.append(TxInput(
+            prev_txid_rev=bytes(32),
+            prev_idx=4294967295,
+            script=b'',
+            sequence=4294967295,
+        ))
+
+    fee = min(input_value, randrange(500))
+    input_value -= fee
+    outputs = []
+    n_outputs = randrange(1, 4)
+    for n in range(n_outputs):
+        value = randrange(input_value + 1)
+        input_value -= value
+        pk_script = coin.hash160_to_P2PKH_script(choice(hash160s))
+        outputs.append(TxOutput(value=value, pk_script=pk_script))
+
+    tx = Tx(version=2, inputs=inputs, outputs=outputs, locktime=0, txid_rev=None, wtxid_rev=None)
+    tx_bytes = tx.serialize()
+    tx_hash = tx_hash_fn(tx_bytes)
+    tx.txid_rev = tx.wtxid_rev = tx_hash
+    for n, output in enumerate(tx.outputs):
+        utxos[(tx_hash, n)] = (coin.hashX_from_script(output.pk_script),
+                               output.value)
+    return tx, tx_hash, tx_bytes
+
+
+class API(MemPoolAPI):
+
+    def __init__(self):
+        self._height = 0
+        self._cached_height = self._db_height = self._height
+        # Create a pool of hash160s.  Map them to their script hashes
+        # Create a bunch of UTXOs paying to those script hashes
+        # Create a bunch of TXs that spend from the UTXO set and create
+        # new outpus, which are added to the UTXO set for later TXs to
+        # spend
+        self.db_utxos = {}
+        self.on_mempool_calls = []
+        self.hashXs = []
+        # Maps of mempool txs from tx_hash to raw and Tx object forms
+        self.raw_txs = {}
+        self.txs = {}
+        self.ordered_adds = []
+
+    def initialize(self, addr_count=100, db_utxo_count=100, mempool_size=50):
+        hash160s = [os.urandom(20) for n in range(addr_count)]
+        self.hashXs = [coin.hash160_to_P2PKH_hashX(hash160)
+                       for hash160 in hash160s]
+        prevouts = [(os.urandom(32), randrange(0, 10))
+                    for n in range (db_utxo_count)]
+        random_value = partial(randrange, coin.VALUE_PER_COIN * 10)
+        self.db_utxos = {prevout: (choice(self.hashXs), random_value())
+                         for prevout in prevouts}
+
+        unspent_utxos = self.db_utxos.copy()
+        for n in range(mempool_size):
+            tx, tx_hash, raw_tx = random_tx(hash160s, unspent_utxos)
+            self.raw_txs[tx_hash] = raw_tx
+            self.txs[tx_hash] = tx
+            self.ordered_adds.append(tx_hash)
+
+    def mempool_utxos(self):
+        utxos = {}
+        for tx_hash, tx in self.txs.items():
+            for n, output in enumerate(tx.outputs):
+                hashX = coin.hashX_from_script(output.pk_script)
+                utxos[(tx_hash, n)] = (hashX, output.value)
+        return utxos
+
+    def mempool_spends(self):
+        return [(input.prev_txid_rev, input.prev_idx)
+                for tx in self.txs.values() for input in tx.inputs
+                if not input.is_generation()]
+
+    def balance_deltas(self):
+        # Return mempool balance deltas indexed by hashX
+        deltas = defaultdict(int)
+        utxos = self.mempool_utxos()
+        for tx_hash, tx in self.txs.items():
+            for n, input in enumerate(tx.inputs):
+                if input.is_generation():
+                    continue
+                prevout = (input.prev_txid_rev, input.prev_idx)
+                if prevout in utxos:
+                    utxos.pop(prevout)
+                else:
+                    hashX, value = self.db_utxos[prevout]
+                    deltas[hashX] -= value
+        for hashX, value in utxos.values():
+            deltas[hashX] += value
+        return deltas
+
+    def spends(self):
+        # Return spends indexed by hashX
+        spends = defaultdict(list)
+        utxos = self.mempool_utxos()
+        for tx_hash, tx in self.txs.items():
+            for n, input in enumerate(tx.inputs):
+                if input.is_generation():
+                    continue
+                prevout = (input.prev_txid_rev, input.prev_idx)
+                if prevout in utxos:
+                    hashX, value = utxos.pop(prevout)
+                else:
+                    hashX, value = self.db_utxos[prevout]
+                spends[hashX].append(prevout)
+        return spends
+
+    def summaries(self):
+        # Return lists of (tx_hash, fee, has_unconfirmed_inputs) by hashX
+        summaries = defaultdict(list)
+        utxos = self.mempool_utxos()
+        for tx_hash, tx in self.txs.items():
+            fee = 0
+            hashXs = set()
+            has_ui = False
+            for n, input in enumerate(tx.inputs):
+                if input.is_generation():
+                    continue
+                has_ui = has_ui or (input.prev_txid_rev in self.txs)
+                prevout = (input.prev_txid_rev, input.prev_idx)
+                if prevout in utxos:
+                    hashX, value = utxos[prevout]
+                else:
+                    hashX, value = self.db_utxos[prevout]
+                hashXs.add(hashX)
+                fee += value
+
+            for output in tx.outputs:
+                hashXs.add(coin.hashX_from_script(output.pk_script))
+                fee -= output.value
+
+            summary = (tx_hash, fee, has_ui)
+            for hashX in hashXs:
+                summaries[hashX].append(summary)
+        return summaries
+
+    def touched(self, tx_hashes):
+        touched = set()
+        utxos = self.mempool_utxos()
+        for tx_hash in tx_hashes:
+            tx = self.txs[tx_hash]
+            for n, input in enumerate(tx.inputs):
+                if input.is_generation():
+                    continue
+                prevout = (input.prev_txid_rev, input.prev_idx)
+                if prevout in utxos:
+                    hashX, value = utxos[prevout]
+                else:
+                    hashX, value = self.db_utxos[prevout]
+                touched.add(hashX)
+
+            for output in tx.outputs:
+                touched.add(coin.hashX_from_script(output.pk_script))
+        return touched
+
+    def UTXOs(self):
+        # Return lists of UTXO 5-tuples by hashX
+        utxos = defaultdict(list)
+        for tx_hash, tx in self.txs.items():
+            for n, output in enumerate(tx.outputs):
+                hashX = coin.hashX_from_script(output.pk_script)
+                utxos[hashX].append((-1, n, tx_hash, 0, output.value))
+        return utxos
+
+    async def height(self):
+        await sleep(0)
+        self._cached_height = self._height
+        return self._height
+
+    def db_height(self):
+        return self._db_height
+
+    async def db_height_changed(self) -> None:
+        await asyncio.Event().wait()  # wait forever: fall back to polling
+
+    async def daemon_height_changed(self) -> None:
+        await asyncio.Event().wait()  # wait forever: fall back to polling
+
+    def cached_height(self):
+        return self._cached_height
+
+    async def mempool_txids_hum(self):
+        await sleep(0)
+        return [hash_to_hex_str(hash) for hash in self.txs]
+
+    async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]:
+        await sleep(0)
+        txids_rev = [hex_str_to_hash(hex_hash) for hex_hash in txids_hum]
+        return [self.raw_txs.get(txid_rev) for txid_rev in txids_rev]
+
+    async def lookup_utxos(self, prevouts):
+        await sleep(0)
+        return [self.db_utxos.get(prevout) for prevout in prevouts]
+
+    async def on_mempool(self, *, touched_hashxs, touched_outpoints, height):
+        self.on_mempool_calls.append((touched_hashxs, height))
+        await sleep(0)
+
+
+class DropAPI(API):
+
+    def __init__(self, drop_count):
+        super().__init__()
+        self.drop_count = drop_count
+        self.dropped = False
+
+    async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]:
+        if not self.dropped:
+            self.dropped = True
+            for hash in self.ordered_adds[-self.drop_count:]:
+                del self.raw_txs[hash]
+                del self.txs[hash]
+        return await super().raw_transactions(txids_hum)
+
+
+def in_caplog(caplog, message):
+    return any(message in record.message for record in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_keep_synchronized(caplog):
+    api = API()
+    mempool = MemPool(coin, api, refresh_secs=0.01)
+    event = Event()
+    with caplog.at_level(logging.INFO):
+        async with OldTaskGroup() as group:
+            await group.spawn(mempool.keep_synchronized, event)
+            # do two iterations of _refresh_hashes, then cancel:
+            await event.wait()
+            await event.wait()
+            await group.cancel_remaining()
+
+    assert in_caplog(caplog, 'beginning processing of daemon mempool')
+    assert in_caplog(caplog, 'compact fee histogram')
+    assert in_caplog(caplog, 'synced in ')
+    assert in_caplog(caplog, '0 txs')
+    assert in_caplog(caplog, 'touching 0 addresses')
+    assert not in_caplog(caplog, 'txs dropped')
+
+
+@pytest.mark.asyncio
+async def test_balance_delta():
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api)
+    event = Event()
+    async with OldTaskGroup() as group:
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        await group.cancel_remaining()
+
+    # Check the default dict is handled properly
+    prior_len = len(mempool.hashXs)
+    assert await mempool.balance_delta(os.urandom(HASHX_LEN)) == 0
+    assert prior_len == len(mempool.hashXs)
+
+    # Test all hashXs
+    deltas = api.balance_deltas()
+    for hashX in api.hashXs:
+        expected = deltas.get(hashX, 0)
+        assert await mempool.balance_delta(hashX) == expected
+
+
+@pytest.mark.asyncio
+async def test_compact_fee_histogram():
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api)
+    event = Event()
+    async with OldTaskGroup() as group:
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        await group.cancel_remaining()
+
+    histogram = await mempool.compact_fee_histogram()
+    assert histogram == []
+    bin_size = 1000
+    mempool._update_histogram(bin_size)
+    histogram = await mempool.compact_fee_histogram()
+    assert len(histogram) > 0
+    rates, sizes = zip(*histogram)
+    assert all(rates[n] < rates[n - 1] for n in range(1, len(rates)))
+
+
+def test_compress_histogram():
+    histogram = {
+        10: 100_000,
+        11: 1_000,
+        12: 10_000_000,
+        13: 1_000,
+        14: 1_000,
+        15: 1_000,
+        16: 1_000,
+        17: 1_000,
+        18: 900_000,
+        19: 1_000,
+        20: 1_000,
+        21: 75_000,
+        22: 1_000,
+    }
+    compact = MemPool._compress_histogram(histogram, bin_size=100_000)
+    assert compact == [(19, 78000), (18, 900000), (13, 5000), (12, 10000000)]
+    compact = MemPool._compress_histogram(histogram, bin_size=30_000)
+    assert compact == [(22, 1000), (21, 75000), (19, 2000), (18, 900000), (13, 5000), (12, 10000000), (10, 101000)]
+
+    histogram = {
+        1.0: 10_000_000,
+        1.1: 30_000,
+        1.2: 40_000,
+        10: 500_000,
+        10.1: 1_000,
+        11: 50_000,
+    }
+    compact = MemPool._compress_histogram(histogram, bin_size=100_000)
+    assert compact == [(10.1, 51000), (10, 500000), (1.1, 70000), (1.0, 10000000)]
+
+
+@pytest.mark.asyncio
+async def test_potential_spends():
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api)
+    event = Event()
+    async with OldTaskGroup() as group:
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        await group.cancel_remaining()
+
+    # Check the default dict is handled properly
+    prior_len = len(mempool.hashXs)
+    assert await mempool.potential_spends(os.urandom(HASHX_LEN)) == set()
+    assert prior_len == len(mempool.hashXs)
+
+    # Test all hashXs
+    spends = api.spends()
+    for hashX in api.hashXs:
+        ps = await mempool.potential_spends(hashX)
+        assert all(spend in ps for spend in spends[hashX])
+
+
+async def _test_summaries(mempool, api):
+    # Test all hashXs
+    summaries = api.summaries()
+    for hashX in api.hashXs:
+        mempool_result = await mempool.transaction_summaries(hashX)
+        mempool_result = [(item.txid_rev, item.fee, item.has_unconfirmed_inputs)
+                          for item in mempool_result]
+        our_result = summaries.get(hashX, [])
+        assert set(our_result) == set(mempool_result)
+
+
+@pytest.mark.asyncio
+async def test_transaction_summaries(caplog):
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api)
+    event = Event()
+    with caplog.at_level(logging.INFO):
+        async with OldTaskGroup() as group:
+            await group.spawn(mempool.keep_synchronized, event)
+            await event.wait()
+            await group.cancel_remaining()
+
+    # Check the default dict is handled properly
+    prior_len = len(mempool.hashXs)
+    assert await mempool.transaction_summaries(os.urandom(HASHX_LEN)) == []
+    assert prior_len == len(mempool.hashXs)
+
+    await _test_summaries(mempool, api)
+    assert not in_caplog(caplog, 'txs dropped')
+
+
+@pytest.mark.asyncio
+async def test_unordered_UTXOs():
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api)
+    event = Event()
+    async with OldTaskGroup() as group:
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        await group.cancel_remaining()
+
+    # Check the default dict is handled properly
+    prior_len = len(mempool.hashXs)
+    assert await mempool.unordered_UTXOs(os.urandom(HASHX_LEN)) == []
+    assert prior_len == len(mempool.hashXs)
+
+    # Test all hashXs
+    utxos = api.UTXOs()
+    for hashX in api.hashXs:
+        mempool_result = await mempool.unordered_UTXOs(hashX)
+        our_result = utxos.get(hashX, [])
+        assert set(our_result) == {
+            dataclasses.astuple(mr)
+            for mr in mempool_result
+        }
+
+
+@pytest.mark.asyncio
+async def test_mempool_removals():
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api, refresh_secs=0.01)
+    event = Event()
+    async with OldTaskGroup() as group:
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        # Remove half the TXs from the mempool
+        start = len(api.ordered_adds) // 2
+        for tx_hash in api.ordered_adds[start:]:
+            del api.txs[tx_hash]
+            del api.raw_txs[tx_hash]
+        await event.wait()
+        await _test_summaries(mempool, api)
+        # Removed hashXs should have key destroyed
+        assert all(mempool.hashXs.values())
+        # Remove the rest
+        api.txs.clear()
+        api.raw_txs.clear()
+        await event.wait()
+        await _test_summaries(mempool, api)
+        assert not mempool.hashXs
+        assert not mempool.txs
+        await group.cancel_remaining()
+
+
+@pytest.mark.asyncio
+async def test_daemon_drops_txs():
+    # Tests things work if the daemon drops some transactions between
+    # returning their hashes and the mempool requesting the raw txs
+    api = DropAPI(10)
+    api.initialize()
+    mempool = MemPool(coin, api, refresh_secs=0.01)
+    event = Event()
+    async with OldTaskGroup() as group:
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        await _test_summaries(mempool, api)
+        await group.cancel_remaining()
+
+
+@pytest.mark.asyncio
+async def test_notifications(caplog):
+    # Tests notifications over a cycle of:
+    # 1) A first batch of txs come in
+    # 2) A second batch of txs come in
+    # 3) A block comes in confirming the first batch only
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api, refresh_secs=0.001, log_status_secs=0)
+    event = Event()
+
+    n = len(api.ordered_adds) // 2
+    raw_txs = api.raw_txs.copy()
+    txs = api.txs.copy()
+    first_hashes = api.ordered_adds[:n]
+    first_touched = api.touched(first_hashes)
+    second_hashes = api.ordered_adds[n:]
+    second_touched = api.touched(second_hashes)
+
+    caplog.set_level(logging.DEBUG)
+
+    async with OldTaskGroup() as group:
+        # First batch enters the mempool
+        api.raw_txs = {hash: raw_txs[hash] for hash in first_hashes}
+        api.txs = {hash: txs[hash] for hash in first_hashes}
+        first_utxos = api.mempool_utxos()
+        first_spends = api.mempool_spends()
+        await group.spawn(mempool.keep_synchronized, event)
+        await event.wait()
+        assert len(api.on_mempool_calls) == 1
+        touched, height = api.on_mempool_calls[0]
+        assert height == api._height == api._db_height == api._cached_height
+        assert touched == first_touched
+        # Second batch enters the mempool
+        api.raw_txs = raw_txs
+        api.txs = txs
+        await event.wait()
+        assert len(api.on_mempool_calls) == 2
+        touched, height = api.on_mempool_calls[1]
+        assert height == api._height == api._db_height == api._cached_height
+        # Touched is incremental
+        assert touched == second_touched
+        # Block found; first half confirm
+        new_height = 2
+        api._height = new_height
+        api.raw_txs = {hash: raw_txs[hash] for hash in second_hashes}
+        api.txs = {hash: txs[hash] for hash in second_hashes}
+        # Delay the DB update
+        assert not in_caplog(caplog, 'waiting for DB to sync')
+        async with ignore_after(max(mempool.refresh_secs * 2, 0.5)):
+            await event.wait()
+        assert in_caplog(caplog, 'waiting for DB to sync')
+        assert len(api.on_mempool_calls) == 2
+        assert not event.is_set()
+        assert api._height == api._cached_height == new_height
+        assert touched == second_touched
+        # Now update the DB
+        api.db_utxos.update(first_utxos)
+        api._db_height = new_height
+        for spend in first_spends:
+            del api.db_utxos[spend]
+        await event.wait()
+        assert len(api.on_mempool_calls) == 3
+        touched, height = api.on_mempool_calls[2]
+        assert height == api._db_height == new_height
+        assert touched == first_touched
+        await group.cancel_remaining()
+
+
+@pytest.mark.asyncio
+async def test_get_recently_added_txs():
+    mempool_size_target = 50
+    api = API()
+    api.initialize(mempool_size=mempool_size_target)
+    mempool = MemPool(coin, api, refresh_secs=0.001, log_status_secs=0)
+    event = Event()
+
+    raw_txs = api.raw_txs.copy()
+    txs = api.txs.copy()
+
+    async with OldTaskGroup() as group:
+        api.raw_txs = {}
+        api.txs = {}
+        await group.spawn(mempool.keep_synchronized, event)
+        for cur_size in range(mempool_size_target):
+            api.raw_txs = {hash: raw_txs[hash] for hash in api.ordered_adds[:cur_size]}
+            api.txs = {hash: txs[hash] for hash in api.ordered_adds[:cur_size]}
+            async with ignore_after(max(mempool.refresh_secs * 2, 0.5)):
+                await event.wait()
+            recent_txs = await mempool.get_recently_added_txs(count=10)
+            start_idx = max(0, cur_size-10)
+            recent_adds = api.ordered_adds[start_idx:cur_size]
+            assert [tx.txid_rev for tx in recent_txs][::-1] == recent_adds
+        await group.cancel_remaining()
+
+
+@pytest.mark.asyncio
+async def test_dropped_txs(caplog):
+    api = API()
+    api.initialize()
+    mempool = MemPool(coin, api)
+    event = Event()
+    # Remove a single TX_HASH that is used in another mempool tx
+    for prev_hash, prev_idx in api.mempool_spends():
+        if prev_hash in api.txs:
+            del api.txs[prev_hash]
+
+    with caplog.at_level(logging.INFO):
+        async with OldTaskGroup() as group:
+            await group.spawn(mempool.keep_synchronized, event)
+            await event.wait()
+            await group.cancel_remaining()
+
+    assert in_caplog(caplog, 'txs dropped')

tests/server/test_notifications.py

@@ -0,0 +1,45 @@
+import pytest
+
+from electrumx.server.controller import Notifications
+
+
+@pytest.mark.asyncio
+async def test_simple_mempool():
+    n = Notifications()
+    notified = []
+    async def notify(*, touched_hashxs, touched_outpoints, height):
+        notified.append((height, touched_hashxs))
+    await n.start(5, notify)
+
+    mtouched = {b'a', b'b'}
+    btouched = {b'b', b'c'}
+    await n.on_mempool(touched_hashxs=mtouched, height=6, touched_outpoints=set())
+    assert notified == [(5, set())]
+    await n.on_block(touched_hashxs=btouched, height=6, touched_outpoints=set())
+    assert notified == [(5, set()), (6, set.union(mtouched, btouched))]
+
+
+@pytest.mark.asyncio
+async def test_enter_mempool_quick_blocks_2():
+    n = Notifications()
+    notified = []
+    async def notify(*, touched_hashxs, touched_outpoints, height):
+        notified.append((height, touched_hashxs))
+    await n.start(5, notify)
+
+    # Suppose a gets in block 6 and blocks 7,8 found right after and
+    # the block processor processes them together.
+    await n.on_mempool(touched_hashxs={b'a'}, height=5, touched_outpoints=set())
+    assert notified == [(5, set()), (5, {b'a'})]
+    # Mempool refreshes with daemon on block 6
+    await n.on_mempool(touched_hashxs={b'a'}, height=6, touched_outpoints=set())
+    assert notified == [(5, set()), (5, {b'a'})]
+    # Blocks 6, 7 processed together
+    await n.on_block(touched_hashxs={b'a', b'b'}, height=7, touched_outpoints=set())
+    assert notified == [(5, set()), (5, {b'a'})]
+    # Then block 8 processed
+    await n.on_block(touched_hashxs={b'c'}, height=8, touched_outpoints=set())
+    assert notified == [(5, set()), (5, {b'a'})]
+    # Now mempool refreshes
+    await n.on_mempool(touched_hashxs=set(), height=8, touched_outpoints=set())
+    assert notified == [(5, set()), (5, {b'a'}), (8, {b'a', b'b', b'c'})]

tests/server/test_storage.py

@@ -0,0 +1,173 @@
+import pytest
+import os
+
+from electrumx.server.storage import Storage, db_class
+from electrumx.lib.util import subclasses
+
+# Find out which db engines to test
+# Those that are not installed will be skipped
+db_engines = []
+for c in subclasses(Storage):
+    try:
+        c.import_module()
+    except ImportError:
+        db_engines.append(pytest.param(c.__name__, marks=pytest.mark.skip))
+    else:
+        db_engines.append(c.__name__)
+
+
+@pytest.fixture(params=db_engines)
+def db(tmpdir, request):
+    cwd = os.getcwd()
+    os.chdir(str(tmpdir))
+    db = db_class(request.param)("db", False)
+    yield db
+    os.chdir(cwd)
+    db.close()
+
+
+def test_put_get(db):
+    db.put(b"x", b"y")
+    assert db.get(b"x") == b"y"
+
+
+def test_batch(db):
+    """Batch updates are only committed *after* the context manager exits."""
+    db.put(b"a", b"1")
+    with db.write_batch() as b:
+        b.put(b"a", b"2")
+        assert db.get(b"a") == b"1"
+    assert db.get(b"a") == b"2"
+
+
+def test_batch_exception(db):
+    """Batch updates are *not* committed if the context manager exits with an exception."""
+    class FakeError(Exception): pass
+
+    db.put(b"a", b"1")
+    try:
+        with db.write_batch() as b:
+            b.put(b"a", b"2")
+            raise FakeError
+    except FakeError:
+        pass
+    else:
+        assert False
+    assert db.get(b"a") == b"1"
+
+
+def test_batch_delete(db):
+    db.put(b"a1", b"")
+    db.put(b"a2", b"")
+    db.put(b"a4", b"")
+    db.put(b"a5", b"")
+    assert list(db.iterator()) == [(b"a1", b""), (b"a2", b""), (b"a4", b""), (b"a5", b"")]
+    with db.write_batch() as b:
+        b.delete(b"a2")
+        b.delete(b"a3")
+        b.delete(b"a4")
+    assert list(db.iterator()) == [(b"a1", b""), (b"a5", b"")]
+
+
+def test_iterator(db):
+    """
+    The iterator should contain all key/value pairs starting with prefix
+    ordered by key.
+    """
+    for i in range(5):
+        db.put(b"abc" + str.encode(str(i)), str.encode(str(i)))
+    db.put(b"abc", b"")
+    db.put(b"a", b"xyz")
+    db.put(b"abd", b"x")
+    assert list(db.iterator(prefix=b"abc")) == [(b"abc", b"")] + [
+            (b"abc" + str.encode(str(i)), str.encode(str(i))) for
+            i in range(5)
+        ]
+
+
+def test_iterator_reverse(db):
+    for i in range(5):
+        db.put(b"abc" + str.encode(str(i)), str.encode(str(i)))
+    db.put(b"a", b"xyz")
+    db.put(b"abd", b"x")
+    assert list(db.iterator(prefix=b"abc", reverse=True)) == [
+            (b"abc" + str.encode(str(i)), str.encode(str(i))) for
+            i in reversed(range(5))
+        ]
+
+
+def test_iterator_seek(db):
+    db.put(b"first-key1", b"val")
+    db.put(b"first-key2", b"val")
+    db.put(b"first-key3", b"val")
+    db.put(b"key-1", b"value-1")
+    db.put(b"key-5", b"value-5")
+    db.put(b"key-3", b"value-3")
+    db.put(b"key-8", b"value-8")
+    db.put(b"key-2", b"value-2")
+    db.put(b"key-4", b"value-4")
+    db.put(b"last-key1", b"val")
+    db.put(b"last-key2", b"val")
+    db.put(b"last-key3", b"val")
+    # forward-iterate, key present, no prefix
+    it = db.iterator()
+    it.seek(b"key-4")
+    assert list(it) == [(b"key-4", b"value-4"), (b"key-5", b"value-5"), (b"key-8", b"value-8"),
+                        (b"last-key1", b"val"), (b"last-key2", b"val"), (b"last-key3", b"val")]
+    # forward-iterate, key present
+    it = db.iterator(prefix=b"key-")
+    it.seek(b"key-4")
+    assert list(it) == [(b"key-4", b"value-4"), (b"key-5", b"value-5"),
+                        (b"key-8", b"value-8")]
+    # forward-iterate, key missing
+    it = db.iterator(prefix=b"key-")
+    it.seek(b"key-6")
+    assert list(it) == [(b"key-8", b"value-8")]
+    # forward-iterate, after last prefix
+    it = db.iterator(prefix=b"key-")
+    it.seek(b"key-9")
+    assert list(it) == []
+    # forward-iterate, after last, no prefix
+    it = db.iterator()
+    it.seek(b"z")
+    assert list(it) == []
+    # forward-iterate, no such prefix
+    it = db.iterator(prefix=b"key---")
+    it.seek(b"key---5")
+    assert list(it) == []
+    # forward-iterate, seek outside prefix
+    it = db.iterator(prefix=b"key-")
+    it.seek(b"last-key2")
+    assert list(it) == []
+    # reverse-iterate, key present
+    it = db.iterator(prefix=b"key-", reverse=True)
+    it.seek(b"key-4")
+    assert list(it) == [(b"key-3", b"value-3"), (b"key-2", b"value-2"), (b"key-1", b"value-1")]
+    # reverse-iterate, key missing
+    it = db.iterator(prefix=b"key-", reverse=True)
+    it.seek(b"key-7")
+    assert list(it) == [(b"key-5", b"value-5"), (b"key-4", b"value-4"), (b"key-3", b"value-3"),
+                        (b"key-2", b"value-2"), (b"key-1", b"value-1")]
+    # reverse-iterate, before first prefix
+    it = db.iterator(prefix=b"key-", reverse=True)
+    it.seek(b"key-0")
+    assert list(it) == []
+    # reverse-iterate, before first, no prefix
+    it = db.iterator(reverse=True)
+    it.seek(b"a")
+    assert list(it) == []
+    # reverse-iterate, no such prefix
+    it = db.iterator(prefix=b"key---", reverse=True)
+    it.seek(b"key---5")
+    assert list(it) == []
+    # reverse-iterate, seek outside prefix
+    it = db.iterator(prefix=b"key-", reverse=True)
+    it.seek(b"first-key2")
+    assert list(it) == []
+
+
+def test_close(db):
+    db.put(b"a", b"b")
+    db.close()
+    db = db_class(db.__class__.__name__)("db", False)
+    assert db.get(b"a") == b"b"