Add integration tests against a Palladium regtest node in Docker

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.github/workflows/ci.yml

@@ -104,3 +104,38 @@ jobs:
 
       - name: Smoke test
         run: contrib/docker-smoke-test.sh plm-electrumx
+
+  integration:
+    # ElectrumX against a real Palladium node in regtest mode (tests/integration)
+    name: integration
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.14'
+          cache: 'pip'
+          cache-dependency-path: 'pyproject.toml'
+
+      - name: Install system packages
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get install -yq libleveldb-dev
+
+      - name: Install Python deps
+        run: |
+          pip install pytest
+          pip install ".[leveldb]"
+
+      - name: Start the regtest node
+        run: docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+
+      - name: Run integration tests
+        env:
+          PLM_TEST_DAEMON_URL: http://plmtest:plmtest@127.0.0.1:28443/
+        run: pytest tests/integration
+
+      - name: Node log
+        if: failure()
+        run: docker compose -f tests/integration/docker-compose.yml logs palladiumd | tail -100

CLAUDE.md

@@ -16,8 +16,14 @@ pytest                                   # all tests (what CI runs, with --cov=e
 pytest tests/lib/test_palladium.py       # single file
 pytest tests/server/test_mempool.py::test_name   # single test
 pycodestyle --max-line-length=140 src    # the only lint, enforced in CI
+
 cd docs && make html                     # docs (Sphinx)
 
+# integration tests: ElectrumX against a real Palladium regtest node in Docker (skipped unless PLM_TEST_DAEMON_URL is set)
+docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+PLM_TEST_DAEMON_URL=http://plmtest:plmtest@127.0.0.1:28443/ pytest tests/integration
+docker compose -f tests/integration/docker-compose.yml down -v
+
 DB_ENGINE=rocksdb DB_DIRECTORY=... DAEMON_URL=http://user:pass@host:port/ electrumx_server
 electrumx_rpc getinfo                    # needs rpc:// in SERVICES (the Docker image does not enable it)
 contrib/docker-smoke-test.sh             # after `docker build -t plm-electrumx .`: checks the SSL handling of the entrypoint
@@ -41,6 +47,8 @@ DB schema changed in 2.0 with no migration; changing on-disk format requires a f
 
 ## Tests
 
+Integration tests (`tests/integration`): `node/Dockerfile` + `node/palladium.conf` run the Palladium node (official release binaries, signature checked) in regtest, `conftest.py` starts `electrumx_server` on an empty DB against it and the tests mine blocks and check headers, balances, mempool, history, merkle proofs and broadcast through the Electrum protocol. Regtest coinbase maturity is 120 blocks, bech32 hrp is `rplm`. The CI job is `integration`. Bump `VERSION` in the Dockerfile to test against a newer node.
+
 `tests/lib` (Palladium params, addresses, hashes, scripts), `tests/server` (env, daemon, mempool, storage, API), plus fixture-driven `tests/test_blocks.py` / `test_transactions.py` using JSON files named `<coin>_<net>_<id>.json` in `tests/blocks` and `tests/transactions` (real mainnet data from a synced node; tx vout values in satoshis). `test_storage.py` needs both DB engines installed. Bech32 addresses are not supported by `Coin.address_to_hashX` (base58 only). Fixtures were taken from a synced node (`getblock <hash> 0|1`, `getrawtransaction <txid> true <blockhash>`, run via `docker compose exec palladiumd palladium-cli ...` in the node's project); vout values converted to satoshis. Ports, SSL, RPC credentials and donation address are documented in `DEPLOY.md`.
 
 ## Repository and workflow

tests/integration/conftest.py

@@ -0,0 +1,180 @@
+'''Integration tests: ElectrumX against a real Palladium node in regtest mode.
+
+They are skipped unless PLM_TEST_DAEMON_URL is set, so a plain `pytest` does not need a node. To run them:
+
+    docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+    PLM_TEST_DAEMON_URL=http://plmtest:plmtest@127.0.0.1:28443/ pytest tests/integration
+
+PLM_TEST_DB_ENGINE (default leveldb) selects the database engine of the server under test.
+'''
+import base64
+import json
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from hashlib import sha256
+from pathlib import Path
+
+import pytest
+
+DAEMON_URL = os.environ.get('PLM_TEST_DAEMON_URL')
+REPO_ROOT = Path(__file__).resolve().parents[2]
+MIN_HEIGHT = 130
+
+
+def pytest_collection_modifyitems(config, items):
+    if DAEMON_URL:
+        return
+    skip = pytest.mark.skip(reason='PLM_TEST_DAEMON_URL not set (no regtest node)')
+    here = Path(__file__).parent
+    for item in items:
+        if here in Path(str(item.fspath)).parents:
+            item.add_marker(skip)
+
+
+class Node:
+    '''Minimal JSON-RPC client of the regtest node.'''
+
+    def __init__(self, url):
+        parts = urllib.parse.urlsplit(url)
+        auth = base64.b64encode(f'{parts.username}:{parts.password}'.encode()).decode()
+        self.url = f'{parts.scheme}://{parts.hostname}:{parts.port}/'
+        self.headers = {'Authorization': f'Basic {auth}', 'Content-Type': 'application/json'}
+
+    def call(self, method, *params, wallet=None):
+        url = self.url + (f'wallet/{wallet}' if wallet is not None else '')
+        body = json.dumps({'jsonrpc': '1.0', 'id': 'test', 'method': method, 'params': list(params)}).encode()
+        req = urllib.request.Request(url, body, self.headers)
+        try:
+            with urllib.request.urlopen(req, timeout=60) as resp:
+                reply = json.load(resp)
+        except urllib.error.HTTPError as e:  # the node answers errors with HTTP 500 and a JSON body
+            reply = json.load(e)
+        if reply.get('error'):
+            raise RuntimeError(f'{method}: {reply["error"]}')
+        return reply['result']
+
+    def mine(self, n, address):
+        return self.call('generatetoaddress', n, address)
+
+
+@pytest.fixture(scope='session')
+def node():
+    node = Node(DAEMON_URL)
+    deadline = time.monotonic() + 60
+    while True:
+        try:
+            node.call('getblockcount')
+            break
+        except Exception:
+            if time.monotonic() > deadline:
+                raise
+            time.sleep(1)
+    return node
+
+
+@pytest.fixture(scope='session')
+def wallet(node):
+    '''Name of a loaded wallet (the node may or may not create a default one)'''
+    if not node.call('listwallets'):
+        node.call('createwallet', 'test')
+    return node.call('listwallets')[0]
+
+
+def new_address(node, wallet):
+    return node.call('getnewaddress', wallet=wallet)
+
+
+def scripthash(node, address):
+    '''Electrum scripthash of an address of the node's wallet: sha256 of the scriptPubKey, byte-reversed'''
+    script = bytes.fromhex(node.call('getaddressinfo', address)['scriptPubKey'])
+    return sha256(script).digest()[::-1].hex()
+
+
+class ElectrumClient:
+    '''Newline-delimited JSON-RPC over TCP, one request at a time (notifications are skipped).'''
+
+    def __init__(self, port):
+        self.sock = socket.create_connection(('127.0.0.1', port), timeout=30)
+        self.file = self.sock.makefile('rwb')
+        self.next_id = 0
+
+    def call(self, method, *params):
+        self.next_id += 1
+        msg = {'jsonrpc': '2.0', 'id': self.next_id, 'method': method, 'params': list(params)}
+        self.file.write(json.dumps(msg).encode() + b'\n')
+        self.file.flush()
+        while True:
+            reply = json.loads(self.file.readline())
+            if reply.get('id') == self.next_id:
+                if 'error' in reply:
+                    raise RuntimeError(f'{method}: {reply["error"]}')
+                return reply['result']
+
+    def close(self):
+        self.file.close()
+        self.sock.close()
+
+
+def free_port():
+    with socket.socket() as s:
+        s.bind(('127.0.0.1', 0))
+        return s.getsockname()[1]
+
+
+@pytest.fixture(scope='session')
+def electrumx(node, wallet):
+    '''Starts electrumx_server on an empty database, against the regtest node. Yields the TCP port.'''
+    # long enough to have mature coinbases to spend (COINBASE_MATURITY is 120 on Palladium)
+    miner = new_address(node, wallet)
+    if node.call('getblockcount') < MIN_HEIGHT:
+        node.mine(MIN_HEIGHT - node.call('getblockcount'), miner)
+    port = free_port()
+    with tempfile.TemporaryDirectory() as db_dir:
+        env = dict(os.environ,
+                   COIN='Palladium', NET='regtest',
+                   DB_ENGINE=os.environ.get('PLM_TEST_DB_ENGINE', 'leveldb'), DB_DIRECTORY=db_dir,
+                   DAEMON_URL=DAEMON_URL, SERVICES=f'tcp://127.0.0.1:{port}',
+                   PEER_DISCOVERY='off', PEER_ANNOUNCE='', LOG_LEVEL='info')
+        log = open(os.path.join(db_dir, 'electrumx.log'), 'w+')
+        proc = subprocess.Popen([sys.executable, str(REPO_ROOT / 'electrumx_server')], cwd=REPO_ROOT, env=env,
+                                stdout=log, stderr=subprocess.STDOUT)
+        try:
+            deadline = time.monotonic() + 60
+            while True:
+                if proc.poll() is not None:
+                    log.seek(0)
+                    pytest.fail(f'electrumx_server exited with {proc.returncode}:\n{log.read()}')
+                try:
+                    socket.create_connection(('127.0.0.1', port), timeout=1).close()
+                    break
+                except OSError:
+                    if time.monotonic() > deadline:
+                        log.seek(0)
+                        pytest.fail(f'electrumx_server did not start:\n{log.read()}')
+                    time.sleep(0.5)
+            yield port
+        finally:
+            proc.terminate()
+            try:
+                proc.wait(timeout=30)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+            log.close()
+
+
+def wait_for(predicate, what, timeout=60):
+    deadline = time.monotonic() + timeout
+    while True:
+        value = predicate()
+        if value:
+            return value
+        if time.monotonic() > deadline:
+            raise AssertionError(f'timeout waiting for {what}')
+        time.sleep(0.5)

tests/integration/docker-compose.yml

@@ -0,0 +1,17 @@
+# Palladium node in regtest mode for the integration tests (tests/integration). From the root of the repository:
+#
+#   docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+#   pytest tests/integration
+#   docker compose -f tests/integration/docker-compose.yml down -v
+services:
+  palladiumd:
+    build: node
+    image: plm-test-node
+    # published on the loopback only
+    ports:
+      - "127.0.0.1:28443:28443"
+    healthcheck:
+      test: ["CMD", "palladium-cli", "-regtest", "-rpcuser=plmtest", "-rpcpassword=plmtest", "getblockcount"]
+      interval: 2s
+      timeout: 5s
+      retries: 30

tests/integration/node/Dockerfile

@@ -0,0 +1,52 @@
+# Palladium node in regtest mode, to run the integration tests of this repository against a real daemon.
+# It runs the official release binaries (checked against the clearsigned SHA256SUMS.asc, which is checked against
+# the pinned release key), with the palladium.conf next to this file.
+#
+#   docker compose -f tests/integration/docker-compose.yml up -d --build --wait
+#
+# Not for production: the RPC credentials are public and the chain is a throwaway regtest chain.
+
+FROM ubuntu:22.04 AS fetch
+
+ARG VERSION=2.1.0
+# Fingerprint of palladium-release-key.asc, published in the release notes of the node
+ARG RELEASE_FPR=70D5C2A2F6C5A128CD9223F227AB4AEDBF5D8BC1
+# Set automatically by BuildKit: amd64, arm64 or arm
+ARG TARGETARCH
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates curl gnupg \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /dl
+RUN set -eux; \
+    case "${TARGETARCH}" in \
+      amd64) T=x86_64-linux-gnu ;; \
+      arm64) T=aarch64-linux-gnu ;; \
+      arm)   T=arm-linux-gnueabihf ;; \
+      *) echo "unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
+    esac; \
+    URL=https://github.com/palladium-coin/palladiumcore/releases/download/v${VERSION}; \
+    F=palladium-${VERSION}-${T}.tar.gz; \
+    for f in "$F" SHA256SUMS.asc palladium-release-key.asc; do curl -fsSLO "$URL/$f"; done; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    gpg --import palladium-release-key.asc; \
+    gpg --with-colons --fingerprint | grep -q "^fpr:::::::::${RELEASE_FPR}:"; \
+    gpg --status-fd 1 --output SHA256SUMS.signed --decrypt SHA256SUMS.asc | grep -q "^\[GNUPG:\] VALIDSIG ${RELEASE_FPR} "; \
+    sha256sum -c --ignore-missing SHA256SUMS.signed; \
+    tar xzf "$F" --strip-components=1 -C /usr/local palladium-${VERSION}/bin/palladiumd palladium-${VERSION}/bin/palladium-cli
+
+FROM ubuntu:22.04
+
+COPY --from=fetch /usr/local/bin/palladiumd /usr/local/bin/palladium-cli /usr/local/bin/
+
+RUN useradd --create-home --uid 1000 palladium \
+ && mkdir /home/palladium/.palladium && chown palladium:palladium /home/palladium/.palladium
+COPY --chown=palladium:palladium palladium.conf /home/palladium/.palladium/palladium.conf
+USER palladium
+
+# regtest RPC and P2P
+EXPOSE 28443 28444
+
+# -daemon=0 and -regtest on the command line: they must not depend on the conf file
+ENTRYPOINT ["palladiumd", "-printtoconsole", "-daemon=0", "-regtest"]

tests/integration/node/palladium.conf

@@ -0,0 +1,20 @@
+# Palladium node for the integration tests: regtest only, throwaway chain, public credentials.
+# Same RPC user and password as tests/integration/conftest.py and docker-compose.yml.
+rpcuser=plmtest
+rpcpassword=plmtest
+
+# what the ElectrumX server needs from the daemon
+server=1
+txindex=1
+fallbackfee=0.0001
+
+# no peers: a lone regtest node
+listen=0
+discover=0
+dnsseed=0
+
+[regtest]
+rpcport=28443
+# reachable from the other containers and, through the published port, from the host (127.0.0.1 only)
+rpcbind=0.0.0.0
+rpcallowip=0.0.0.0/0

tests/integration/test_regtest_node.py

@@ -0,0 +1,79 @@
+'''ElectrumX server end to end against a Palladium regtest node (see conftest.py).'''
+import pytest
+
+from electrumx.lib.coins import PalladiumRegtest
+from tests.integration.conftest import ElectrumClient, new_address, scripthash, wait_for
+
+
+def test_node_matches_coin_parameters(node):
+    info = node.call('getblockchaininfo')
+    assert info['chain'] == 'regtest'
+    genesis = node.call('getblockhash', 0)
+    assert genesis == PalladiumRegtest.GENESIS_HASH
+    # the coin class accepts the genesis block that the node serves
+    raw = bytes.fromhex(node.call('getblock', genesis, 0))
+    assert PalladiumRegtest.genesis_block(raw) == raw[:80] + b'\0'
+
+
+def test_address_prefixes_match_the_node(node, wallet):
+    '''A legacy (base58) address of the node's wallet maps to the hashX of its script'''
+    address = node.call('getnewaddress', '', 'legacy', wallet=wallet)
+    script = bytes.fromhex(node.call('getaddressinfo', address)['scriptPubKey'])
+    assert PalladiumRegtest.address_to_hashX(address) == PalladiumRegtest.hashX_from_script(script)
+
+
+@pytest.fixture
+def client(electrumx):
+    client = ElectrumClient(electrumx)
+    yield client
+    client.close()
+
+
+def test_server_syncs_and_follows_the_chain(node, wallet, electrumx, client):
+    client.call('server.version', 'plm-integration-test', '1.4')
+    wait_for(lambda: client.call('blockchain.headers.subscribe')['height'] == node.call('getblockcount'),
+             'the server to reach the node height')
+    tip = client.call('blockchain.headers.subscribe')
+    assert tip['height'] == node.call('getblockcount')
+
+    node.mine(3, new_address(node, wallet))
+    wait_for(lambda: client.call('blockchain.headers.subscribe')['height'] == tip['height'] + 3,
+             'the server to follow new blocks')
+    header = client.call('blockchain.block.header', tip['height'] + 3)
+    assert PalladiumRegtest.header_hash_rev(bytes.fromhex(header)) == bytes.fromhex(node.call('getbestblockhash'))[::-1]
+
+
+def test_balance_history_and_mempool(node, wallet, electrumx, client):
+    client.call('server.version', 'plm-integration-test', '1.4')
+    miner = new_address(node, wallet)
+    node.mine(1, miner)
+    target = node.call('getnewaddress', '', 'bech32', wallet=wallet)
+    sh = scripthash(node, target)
+
+    # unconfirmed: seen in the server's mempool
+    txid = node.call('sendtoaddress', target, 1.5, wallet=wallet)
+    balance = wait_for(lambda: (b := client.call('blockchain.scripthash.get_balance', sh))['unconfirmed'] and b,
+                       'the transaction to appear in the mempool of the server')
+    assert balance == {'confirmed': 0, 'unconfirmed': 150_000_000}
+    assert client.call('blockchain.scripthash.get_mempool', sh)[0]['tx_hash'] == txid
+
+    # confirmed
+    node.mine(1, miner)
+    balance = wait_for(lambda: (b := client.call('blockchain.scripthash.get_balance', sh))['confirmed'] and b,
+                       'the transaction to be confirmed')
+    assert balance == {'confirmed': 150_000_000, 'unconfirmed': 0}
+    history = client.call('blockchain.scripthash.get_history', sh)
+    assert [h['tx_hash'] for h in history] == [txid]
+    assert history[0]['height'] == node.call('getblockcount')
+
+    # the raw transaction and its merkle proof (needs txindex on the node)
+    raw = client.call('blockchain.transaction.get', txid)
+    assert raw == node.call('getrawtransaction', txid)
+    proof = client.call('blockchain.transaction.get_merkle', txid, history[0]['height'])
+    assert proof['block_height'] == history[0]['height']
+
+    # broadcast through the server
+    other = node.call('getnewaddress', wallet=wallet)
+    funded = node.call('fundrawtransaction', node.call('createrawtransaction', [], {other: 0.1}), wallet=wallet)
+    signed = node.call('signrawtransactionwithwallet', funded['hex'], wallet=wallet)['hex']
+    assert client.call('blockchain.transaction.broadcast', signed) == node.call('decoderawtransaction', signed)['txid']