Add integration tests against a Palladium regtest node in Docker

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tests/integration/conftest.py

@@ -0,0 +1,180 @@
+'''Integration tests: ElectrumX against a real Palladium node in regtest mode.
+
+They are skipped unless PLM_TEST_DAEMON_URL is set, so a plain `pytest` does not need a node. To run them:
+
+    docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+    PLM_TEST_DAEMON_URL=http://plmtest:plmtest@127.0.0.1:28443/ pytest tests/integration
+
+PLM_TEST_DB_ENGINE (default leveldb) selects the database engine of the server under test.
+'''
+import base64
+import json
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from hashlib import sha256
+from pathlib import Path
+
+import pytest
+
+DAEMON_URL = os.environ.get('PLM_TEST_DAEMON_URL')
+REPO_ROOT = Path(__file__).resolve().parents[2]
+MIN_HEIGHT = 130
+
+
+def pytest_collection_modifyitems(config, items):
+    if DAEMON_URL:
+        return
+    skip = pytest.mark.skip(reason='PLM_TEST_DAEMON_URL not set (no regtest node)')
+    here = Path(__file__).parent
+    for item in items:
+        if here in Path(str(item.fspath)).parents:
+            item.add_marker(skip)
+
+
+class Node:
+    '''Minimal JSON-RPC client of the regtest node.'''
+
+    def __init__(self, url):
+        parts = urllib.parse.urlsplit(url)
+        auth = base64.b64encode(f'{parts.username}:{parts.password}'.encode()).decode()
+        self.url = f'{parts.scheme}://{parts.hostname}:{parts.port}/'
+        self.headers = {'Authorization': f'Basic {auth}', 'Content-Type': 'application/json'}
+
+    def call(self, method, *params, wallet=None):
+        url = self.url + (f'wallet/{wallet}' if wallet is not None else '')
+        body = json.dumps({'jsonrpc': '1.0', 'id': 'test', 'method': method, 'params': list(params)}).encode()
+        req = urllib.request.Request(url, body, self.headers)
+        try:
+            with urllib.request.urlopen(req, timeout=60) as resp:
+                reply = json.load(resp)
+        except urllib.error.HTTPError as e:  # the node answers errors with HTTP 500 and a JSON body
+            reply = json.load(e)
+        if reply.get('error'):
+            raise RuntimeError(f'{method}: {reply["error"]}')
+        return reply['result']
+
+    def mine(self, n, address):
+        return self.call('generatetoaddress', n, address)
+
+
+@pytest.fixture(scope='session')
+def node():
+    node = Node(DAEMON_URL)
+    deadline = time.monotonic() + 60
+    while True:
+        try:
+            node.call('getblockcount')
+            break
+        except Exception:
+            if time.monotonic() > deadline:
+                raise
+            time.sleep(1)
+    return node
+
+
+@pytest.fixture(scope='session')
+def wallet(node):
+    '''Name of a loaded wallet (the node may or may not create a default one)'''
+    if not node.call('listwallets'):
+        node.call('createwallet', 'test')
+    return node.call('listwallets')[0]
+
+
+def new_address(node, wallet):
+    return node.call('getnewaddress', wallet=wallet)
+
+
+def scripthash(node, address):
+    '''Electrum scripthash of an address of the node's wallet: sha256 of the scriptPubKey, byte-reversed'''
+    script = bytes.fromhex(node.call('getaddressinfo', address)['scriptPubKey'])
+    return sha256(script).digest()[::-1].hex()
+
+
+class ElectrumClient:
+    '''Newline-delimited JSON-RPC over TCP, one request at a time (notifications are skipped).'''
+
+    def __init__(self, port):
+        self.sock = socket.create_connection(('127.0.0.1', port), timeout=30)
+        self.file = self.sock.makefile('rwb')
+        self.next_id = 0
+
+    def call(self, method, *params):
+        self.next_id += 1
+        msg = {'jsonrpc': '2.0', 'id': self.next_id, 'method': method, 'params': list(params)}
+        self.file.write(json.dumps(msg).encode() + b'\n')
+        self.file.flush()
+        while True:
+            reply = json.loads(self.file.readline())
+            if reply.get('id') == self.next_id:
+                if 'error' in reply:
+                    raise RuntimeError(f'{method}: {reply["error"]}')
+                return reply['result']
+
+    def close(self):
+        self.file.close()
+        self.sock.close()
+
+
+def free_port():
+    with socket.socket() as s:
+        s.bind(('127.0.0.1', 0))
+        return s.getsockname()[1]
+
+
+@pytest.fixture(scope='session')
+def electrumx(node, wallet):
+    '''Starts electrumx_server on an empty database, against the regtest node. Yields the TCP port.'''
+    # long enough to have mature coinbases to spend (COINBASE_MATURITY is 120 on Palladium)
+    miner = new_address(node, wallet)
+    if node.call('getblockcount') < MIN_HEIGHT:
+        node.mine(MIN_HEIGHT - node.call('getblockcount'), miner)
+    port = free_port()
+    with tempfile.TemporaryDirectory() as db_dir:
+        env = dict(os.environ,
+                   COIN='Palladium', NET='regtest',
+                   DB_ENGINE=os.environ.get('PLM_TEST_DB_ENGINE', 'leveldb'), DB_DIRECTORY=db_dir,
+                   DAEMON_URL=DAEMON_URL, SERVICES=f'tcp://127.0.0.1:{port}',
+                   PEER_DISCOVERY='off', PEER_ANNOUNCE='', LOG_LEVEL='info')
+        log = open(os.path.join(db_dir, 'electrumx.log'), 'w+')
+        proc = subprocess.Popen([sys.executable, str(REPO_ROOT / 'electrumx_server')], cwd=REPO_ROOT, env=env,
+                                stdout=log, stderr=subprocess.STDOUT)
+        try:
+            deadline = time.monotonic() + 60
+            while True:
+                if proc.poll() is not None:
+                    log.seek(0)
+                    pytest.fail(f'electrumx_server exited with {proc.returncode}:\n{log.read()}')
+                try:
+                    socket.create_connection(('127.0.0.1', port), timeout=1).close()
+                    break
+                except OSError:
+                    if time.monotonic() > deadline:
+                        log.seek(0)
+                        pytest.fail(f'electrumx_server did not start:\n{log.read()}')
+                    time.sleep(0.5)
+            yield port
+        finally:
+            proc.terminate()
+            try:
+                proc.wait(timeout=30)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+            log.close()
+
+
+def wait_for(predicate, what, timeout=60):
+    deadline = time.monotonic() + timeout
+    while True:
+        value = predicate()
+        if value:
+            return value
+        if time.monotonic() > deadline:
+            raise AssertionError(f'timeout waiting for {what}')
+        time.sleep(0.5)

tests/integration/docker-compose.yml

@@ -0,0 +1,17 @@
+# Palladium node in regtest mode for the integration tests (tests/integration). From the root of the repository:
+#
+#   docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+#   pytest tests/integration
+#   docker compose -f tests/integration/docker-compose.yml down -v
+services:
+  palladiumd:
+    build: node
+    image: plm-test-node
+    # published on the loopback only
+    ports:
+      - "127.0.0.1:28443:28443"
+    healthcheck:
+      test: ["CMD", "palladium-cli", "-regtest", "-rpcuser=plmtest", "-rpcpassword=plmtest", "getblockcount"]
+      interval: 2s
+      timeout: 5s
+      retries: 30

tests/integration/node/Dockerfile

@@ -0,0 +1,52 @@
+# Palladium node in regtest mode, to run the integration tests of this repository against a real daemon.
+# It runs the official release binaries (checked against the clearsigned SHA256SUMS.asc, which is checked against
+# the pinned release key), with the palladium.conf next to this file.
+#
+#   docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+#
+# Not for production: the RPC credentials are public and the chain is a throwaway regtest chain.
+
+FROM ubuntu:22.04 AS fetch
+
+ARG VERSION=2.1.0
+# Fingerprint of palladium-release-key.asc, published in the release notes of the node
+ARG RELEASE_FPR=70D5C2A2F6C5A128CD9223F227AB4AEDBF5D8BC1
+# Set automatically by BuildKit: amd64, arm64 or arm
+ARG TARGETARCH
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates curl gnupg \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /dl
+RUN set -eux; \
+    case "${TARGETARCH}" in \
+      amd64) T=x86_64-linux-gnu ;; \
+      arm64) T=aarch64-linux-gnu ;; \
+      arm)   T=arm-linux-gnueabihf ;; \
+      *) echo "unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    URL=https://github.com/palladium-coin/palladiumcore/releases/download/v${VERSION}; \
+    F=palladium-${VERSION}-${T}.tar.gz; \
+    for f in "$F" SHA256SUMS.asc palladium-release-key.asc; do curl -fsSLO "$URL/$f"; done; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    gpg --import palladium-release-key.asc; \
+    gpg --with-colons --fingerprint | grep -q "^fpr:::::::::${RELEASE_FPR}:"; \
+    gpg --status-fd 1 --output SHA256SUMS.signed --decrypt SHA256SUMS.asc | grep -q "^\[GNUPG:\] VALIDSIG ${RELEASE_FPR} "; \
+    sha256sum -c --ignore-missing SHA256SUMS.signed; \
+    tar xzf "$F" --strip-components=1 -C /usr/local palladium-${VERSION}/bin/palladiumd palladium-${VERSION}/bin/palladium-cli
+
+FROM ubuntu:22.04
+
+COPY --from=fetch /usr/local/bin/palladiumd /usr/local/bin/palladium-cli /usr/local/bin/
+
+RUN useradd --create-home --uid 1000 palladium \
+ && mkdir /home/palladium/.palladium && chown palladium:palladium /home/palladium/.palladium
+COPY --chown=palladium:palladium palladium.conf /home/palladium/.palladium/palladium.conf
+USER palladium
+
+# regtest RPC and P2P
+EXPOSE 28443 28444
+
+# -daemon=0 and -regtest on the command line: they must not depend on the conf file
+ENTRYPOINT ["palladiumd", "-printtoconsole", "-daemon=0", "-regtest"]

tests/integration/node/palladium.conf

@@ -0,0 +1,20 @@
+# Palladium node for the integration tests: regtest only, throwaway chain, public credentials.
+# Same RPC user and password as tests/integration/conftest.py and docker-compose.yml.
+rpcuser=plmtest
+rpcpassword=plmtest
+
+# what the ElectrumX server needs from the daemon
+server=1
+txindex=1
+fallbackfee=0.0001
+
+# no peers: a lone regtest node
+listen=0
+discover=0
+dnsseed=0
+
+[regtest]
+rpcport=28443
+# reachable from the other containers and, through the published port, from the host (127.0.0.1 only)
+rpcbind=0.0.0.0
+rpcallowip=0.0.0.0/0

tests/integration/test_regtest_node.py

@@ -0,0 +1,79 @@
+'''ElectrumX server end to end against a Palladium regtest node (see conftest.py).'''
+import pytest
+
+from electrumx.lib.coins import PalladiumRegtest
+from tests.integration.conftest import ElectrumClient, new_address, scripthash, wait_for
+
+
+def test_node_matches_coin_parameters(node):
+    info = node.call('getblockchaininfo')
+    assert info['chain'] == 'regtest'
+    genesis = node.call('getblockhash', 0)
+    assert genesis == PalladiumRegtest.GENESIS_HASH
+    # the coin class accepts the genesis block that the node serves
+    raw = bytes.fromhex(node.call('getblock', genesis, 0))
+    assert PalladiumRegtest.genesis_block(raw) == raw[:80] + b'\0'
+
+
+def test_address_prefixes_match_the_node(node, wallet):
+    '''A legacy (base58) address of the node's wallet maps to the hashX of its script'''
+    address = node.call('getnewaddress', '', 'legacy', wallet=wallet)
+    script = bytes.fromhex(node.call('getaddressinfo', address)['scriptPubKey'])
+    assert PalladiumRegtest.address_to_hashX(address) == PalladiumRegtest.hashX_from_script(script)
+
+
+@pytest.fixture
+def client(electrumx):
+    client = ElectrumClient(electrumx)
+    yield client
+    client.close()
+
+
+def test_server_syncs_and_follows_the_chain(node, wallet, electrumx, client):
+    client.call('server.version', 'plm-integration-test', '1.4')
+    wait_for(lambda: client.call('blockchain.headers.subscribe')['height'] == node.call('getblockcount'),
+             'the server to reach the node height')
+    tip = client.call('blockchain.headers.subscribe')
+    assert tip['height'] == node.call('getblockcount')
+
+    node.mine(3, new_address(node, wallet))
+    wait_for(lambda: client.call('blockchain.headers.subscribe')['height'] == tip['height'] + 3,
+             'the server to follow new blocks')
+    header = client.call('blockchain.block.header', tip['height'] + 3)
+    assert PalladiumRegtest.header_hash_rev(bytes.fromhex(header)) == bytes.fromhex(node.call('getbestblockhash'))[::-1]
+
+
+def test_balance_history_and_mempool(node, wallet, electrumx, client):
+    client.call('server.version', 'plm-integration-test', '1.4')
+    miner = new_address(node, wallet)
+    node.mine(1, miner)
+    target = node.call('getnewaddress', '', 'bech32', wallet=wallet)
+    sh = scripthash(node, target)
+
+    # unconfirmed: seen in the server's mempool
+    txid = node.call('sendtoaddress', target, 1.5, wallet=wallet)
+    balance = wait_for(lambda: (b := client.call('blockchain.scripthash.get_balance', sh))['unconfirmed'] and b,
+                       'the transaction to appear in the mempool of the server')
+    assert balance == {'confirmed': 0, 'unconfirmed': 150_000_000}
+    assert client.call('blockchain.scripthash.get_mempool', sh)[0]['tx_hash'] == txid
+
+    # confirmed
+    node.mine(1, miner)
+    balance = wait_for(lambda: (b := client.call('blockchain.scripthash.get_balance', sh))['confirmed'] and b,
+                       'the transaction to be confirmed')
+    assert balance == {'confirmed': 150_000_000, 'unconfirmed': 0}
+    history = client.call('blockchain.scripthash.get_history', sh)
+    assert [h['tx_hash'] for h in history] == [txid]
+    assert history[0]['height'] == node.call('getblockcount')
+
+    # the raw transaction and its merkle proof (needs txindex on the node)
+    raw = client.call('blockchain.transaction.get', txid)
+    assert raw == node.call('getrawtransaction', txid)
+    proof = client.call('blockchain.transaction.get_merkle', txid, history[0]['height'])
+    assert proof['block_height'] == history[0]['height']
+
+    # broadcast through the server
+    other = node.call('getnewaddress', wallet=wallet)
+    funded = node.call('fundrawtransaction', node.call('createrawtransaction', [], {other: 0.1}), wallet=wallet)
+    signed = node.call('signrawtransactionwithwallet', funded['hex'], wallet=wallet)['hex']
+    assert client.call('blockchain.transaction.broadcast', signed) == node.call('decoderawtransaction', signed)['txid']