Initial commit: snapshot of spesmilo/electrumx 2.0.0

Unmodified source tree of upstream commit 280cb339aa158e099eb936033eecd15719770651
("Merge pull request #389 from SomberNight/202607_release_2_0",
https://github.com/spesmilo/electrumx, release 2.0.0, 3 July 2026).

All Palladium-specific changes are in the commits that follow.

.github/pull_request_template.md

@@ -0,0 +1,4 @@
+<!-- Note: Our primary focus is supporting Bitcoin, and contributions in that regard are appreciated the most!
+Due to historical reasons, ElectrumX also has support for many altcoins. These will be kept as long as
+maintenance burden can be kept *at a minimum*. When adding a new altcoin, (1) add a unit test (see tests/blocks),
+and (2) make sure the CI tests pass. -->

.github/workflows/ci.yml

@@ -0,0 +1,108 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:  # https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # if a PR gets force pushed, cancel the current run and run the new one
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  unittests:
+    name: unittests
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install system packages
+        run: |
+          sudo apt-get -qq update
+          sudo apt-get install -yq libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev librocksdb-dev libleveldb-dev libboost-all-dev libsodium-dev
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.14'
+          cache: 'pip'
+          cache-dependency-path: 'pyproject.toml'
+
+      - name: Log Python version and Git tag
+        run: |
+          python3 --version
+          git tag
+
+      - name: Install Python deps
+        run: |
+          # test framework -related deps
+          pip install pycodestyle
+          pip install pytest-asyncio
+          pip install pytest-cov
+          pip install Sphinx
+          # e-x itself
+          # - install both leveldb and rocksdb bindings
+          pip install ".[leveldb,rocksdb]"
+          # hashes and other altcoin-specific stuff
+          pip install tribushashm
+          pip install blake256
+          pip install dash_hash
+          pip install git+https://github.com/bitcoinplusorg/x13-hash
+          pip install xevan_hash
+          pip install quark_hash
+          pip install groestlcoin_hash
+          pip install x16r_hash
+          pip install pycryptodomex
+          pip install git+https://github.com/Electra-project/nist5_hash
+          pip install x16rv2_hash
+
+      - name: Run pytest
+        run: pytest --cov=electrumx
+
+  pycodestyle:
+    name: pycodestyle
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.14'
+          cache: 'pip'
+
+      - name: Install pycodestyle
+        run: pip install pycodestyle
+
+      - name: Run pycodestyle
+        run: |
+          pycodestyle \
+            --max-line-length=140 \
+            src
+
+  build_docs:
+    # note: this build task is not related to read-the-docs; it is only here to sanity-check the docs can be built.
+    #       For the read-the-docs build, see .readthedocs.yml
+    #       and the webhook set up in https://github.com/spesmilo/electrumx/settings/hooks
+    name: build_docs
+    runs-on: ubuntu-24.04
+    env:
+      LC_ALL: "C"
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.14'
+          cache: 'pip'
+          cache-dependency-path: 'docs/requirements-docs.txt'
+
+      - name: Install docs deps
+        run: pip install -r docs/requirements-docs.txt
+
+      - name: Build docs
+        run: |
+          cd docs
+          make html

.gitignore

@@ -0,0 +1,19 @@
+**/__pycache__/
+/.cache/
+/.pytest_cache/
+*/*~
+*.#*
+*#
+*~
+/docs/_build/
+/build/
+/dist/
+/venv/
+/.vscode/
+/.mypy_cache/
+/.idea/
+
+/src/electrumx.egg-info/
+/src/e_x.egg-info/
+/electrumx.egg-info/
+/e_x.egg-info/

.readthedocs.yml

@@ -0,0 +1,13 @@
+version: 2
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+   install:
+   - requirements: docs/requirements-docs.txt

LICENCE

@@ -0,0 +1,25 @@
+Copyright (c) 2016-2020, Neil Booth
+Copyright (c) 2020-2026, The Electrum developers
+
+All rights reserved.
+
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -0,0 +1,5 @@
+# note: files listed in this file:
+#       - will be added to the sdist
+#       - but won't be added to the wheel (unless by other means)
+
+graft tests/

README.md

@@ -0,0 +1,31 @@
+# ElectrumX - python electrum server
+
+```
+Licence: MIT
+Original Author: Neil Booth
+Current Maintainers: The Electrum developers
+Language: Python (>= 3.10)
+```
+
+[![Latest PyPI package](https://badge.fury.io/py/e_x.svg)](https://pypi.org/project/e-x/)
+[![Build Status](https://github.com/spesmilo/electrumx/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/spesmilo/electrumx/actions/workflows/ci.yml)
+
+This project is a fork of [kyuupichan/electrumx](https://github.com/kyuupichan/electrumx).
+The original author dropped support for Bitcoin, which we intend to keep.
+
+ElectrumX allows users to run their own Electrum server. It connects to your
+full node and indexes the blockchain, allowing efficient querying of the history of
+arbitrary addresses. The server can be exposed publicly, and joined to the public network
+of servers via peer discovery. As of May 2020, a significant chunk of the public
+Electrum server network runs ElectrumX.
+
+### Documentation
+
+See [readthedocs](https://electrumx-spesmilo.readthedocs.io).
+
+### Releases
+
+Running from the latest git tag is always a safe bet, for which there are also releases on PyPI:
+```
+$ pip install e-x
+```

SECURITY.md

@@ -0,0 +1,3 @@
+# Security Policy
+
+See https://github.com/spesmilo/electrum/blob/master/SECURITY.md

contrib/Dockerfile

@@ -0,0 +1,52 @@
+# ElectrumX for Bitcoin, with RocksDB
+# This is a simple ~minimal example Dockerfile.
+#
+# Build: docker build --pull -t electrumx .
+#        podman build --pull  -t electrumx .
+# Run:   docker run -d --net=host -v ~/electrumx-db:/var/lib/electrumx -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx
+#        podman run -d --net=host -v ~/electrumx-db:/var/lib/electrumx -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx
+# Stop:  docker stop CONTAINER_ID
+#        podman stop CONTAINER_ID
+
+FROM python:3.14-trixie AS builder
+
+WORKDIR /usr/src/app
+
+# Install the libs needed by rocksdb (including development headers)
+RUN apt-get update \
+    && apt-get -y --no-install-recommends install \
+        build-essential \
+        git \
+        librocksdb-dev \
+        pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install ElectrumX
+ARG ELECTRUMX_VERSION=2.0.0
+#ARG ELECTRUMX_VERSION=b8f6e227f251fe5cba96b940ffaa094439636417
+RUN python -m venv venv \
+    && venv/bin/pip install --no-cache-dir \
+        "e_x[rocksdb] @ git+https://github.com/spesmilo/electrumx.git@${ELECTRUMX_VERSION}"
+
+
+FROM python:3.14-slim-trixie
+
+# Install the libs needed by rocksdb (no development headers or statics)
+RUN apt-get update && apt-get -y --no-install-recommends install librocksdb9.10 && rm -rf /var/lib/apt/lists/*
+
+ENV SERVICES="tcp://:50001"
+ENV COIN=Bitcoin
+ENV DB_DIRECTORY=/var/lib/electrumx
+ENV DAEMON_URL="http://username:password@hostname:port/"
+ENV ALLOW_ROOT=true
+ENV DB_ENGINE=rocksdb
+#ENV CACHE_MB=2000
+
+WORKDIR /usr/src/app
+COPY --from=builder /usr/src/app .
+
+VOLUME /var/lib/electrumx
+
+RUN mkdir -p "$DB_DIRECTORY"
+
+CMD ["/usr/src/app/venv/bin/python", "/usr/src/app/venv/bin/electrumx_server"]

contrib/Dockerfile_beefyserver

@@ -0,0 +1,82 @@
+# ElectrumX for Bitcoin, with RocksDB
+# This Dockerfile is intended for modern multi-core servers with at least 16 GB of RAM.
+# We compile our own free-threaded Python, with --disable-gil (which is experimental!), which makes
+# ElectrumX able to truly utilise multiple CPU cores and makes processing blocks significantly faster.
+#
+# Build: docker build -f Dockerfile_beefyserver --pull -t electrumx .
+# Run:   docker run -d --net=host -v ~/electrumx-db:/var/lib/electrumx -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx
+# Stop:  docker stop CONTAINER_ID
+
+FROM debian:trixie
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /usr/src/app
+
+# Install libs needed to build CPython
+RUN apt-get update \
+    && apt-get -y install \
+        git \
+        wget \
+        make \
+        autotools-dev \
+        autoconf \
+        libtool \
+        xz-utils \
+        libssl-dev \
+        libssl3 \
+        openssl \
+        zlib1g-dev \
+        libffi-dev \
+        libncurses5-dev \
+        libsqlite3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install libs needed to build RocksDB
+RUN apt-get update \
+    && apt-get -y --no-install-recommends install \
+        build-essential \
+        git \
+        librocksdb-dev \
+        pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+# build CPython (free-threaded, with --disable-gil)
+ARG PYTHON_VERSION="v3.14.6"
+RUN cd /opt && \
+    git clone https://github.com/python/cpython && \
+    cd cpython/ && \
+    git checkout "${PYTHON_VERSION}" && \
+    ./configure \
+        --prefix="/usr" \
+        --enable-ipv6 \
+        --enable-shared \
+        --disable-gil \
+        -q && \
+    make -s "-j$(nproc)" && \
+    make install
+
+RUN python3 -m pip install --upgrade pip setuptools wheel
+
+# Install ElectrumX
+ARG ELECTRUMX_VERSION=2.0.0
+#ARG ELECTRUMX_VERSION=b8f6e227f251fe5cba96b940ffaa094439636417
+RUN python3 -m pip install \
+    "e_x[rocksdb] @ git+https://github.com/spesmilo/electrumx.git@${ELECTRUMX_VERSION}"
+
+
+ENV SERVICES="tcp://:50001"
+ENV COIN=Bitcoin
+ENV DB_DIRECTORY=/var/lib/electrumx
+ENV DAEMON_URL="http://username:password@hostname:port/"
+ENV ALLOW_ROOT=true
+ENV DB_ENGINE=rocksdb
+ENV CACHE_MB=2000
+
+
+VOLUME /var/lib/electrumx
+
+RUN mkdir -p "$DB_DIRECTORY"
+
+CMD ["electrumx_server"]

contrib/daemontools/env/COIN

@@ -0,0 +1 @@
+Bitcoin

contrib/daemontools/env/DAEMON_URL

@@ -0,0 +1 @@
+http://username:password@host:port/

contrib/daemontools/env/DB_DIRECTORY

@@ -0,0 +1 @@
+/path/to/db/directory

contrib/daemontools/env/ELECTRUMX

@@ -0,0 +1 @@
+/path/to/electrumx_server

contrib/daemontools/env/NET

@@ -0,0 +1 @@
+mainnet

contrib/daemontools/env/USERNAME

@@ -0,0 +1 @@
+electrumx

contrib/daemontools/log/run

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec multilog s500000 n10 /path/to/log/dir

contrib/daemontools/run

@@ -0,0 +1,5 @@
+#!/bin/sh
+echo "Launching ElectrumX server..."
+USERNAME=$(envdir ./env printenv USERNAME)
+ELECTRUMX=$(envdir ./env printenv ELECTRUMX)
+ulimit -n 4000 && exec 2>&1 envdir ./env envuidgid $USERNAME python3 $ELECTRUMX

contrib/query.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Script to query the database for debugging purposes.
+
+Not currently documented; might become easier to use in future.
+'''
+
+import argparse
+import asyncio
+
+from electrumx import Env
+from electrumx.server.db import DB
+from electrumx.lib.hash import hash_to_hex_str, Base58Error
+
+
+async def print_stats(hist_db, utxo_db):
+    count = 0
+    for key in utxo_db.iterator(prefix=b'u', include_value=False):
+        count += 1
+    print(f'UTXO count: {utxos}')
+
+    count = 0
+    for key in utxo_db.iterator(prefix=b'h', include_value=False):
+        count += 1
+    print(f'HashX count: {count}')
+
+    hist = 0
+    hist_len = 0
+    for key, value in hist_db.iterator(prefix=b'H'):
+        hist += 1
+        hist_len += len(value) // 4
+    print(f'History rows {hist:,d} entries {hist_len:,d}')
+
+
+def arg_to_hashX(coin, arg):
+    try:
+        script = bytes.fromhex(arg)
+        print(f'Script: {arg}')
+        return coin.hashX_from_script(script)
+    except ValueError:
+        pass
+
+    try:
+        hashX = coin.address_to_hashX(arg)
+        print(f'Address: {arg}')
+        return hashX
+    except Base58Error:
+        print(f'Ingoring unknown arg: {arg}')
+        return None
+
+
+async def query(args):
+    env = Env()
+    db = DB(env)
+    coin = env.coin
+
+    await db.open_for_serving()
+
+    if not args.scripts:
+        await print_stats(db.hist_db, db.utxo_db)
+        return
+    limit = args.limit
+    for arg in args.scripts:
+        hashX = arg_to_hashX(coin, arg)
+        if not hashX:
+            continue
+        n = None
+        history = await db.limited_history(hashX, limit=limit)
+        for n, (tx_hash, height) in enumerate(history, start=1):
+            print(f'History #{n:,d}: height {height:,d} '
+                  f'tx_hash {hash_to_hex_str(tx_hash)}')
+        if n is None:
+            print('No history found')
+        n = None
+        utxos = await db.all_utxos(hashX)
+        for n, utxo in enumerate(utxos, start=1):
+            print(f'UTXO #{n:,d}: tx_hash {hash_to_hex_str(utxo.txid_rev)} '
+                  f'tx_pos {utxo.tx_pos:,d} height {utxo.height:,d} '
+                  f'value {utxo.value:,d}')
+            if n == limit:
+                break
+        if n is None:
+            print('No UTXOs found')
+        balance = sum(utxo.value for utxo in utxos)
+        print(f'Balance: {coin.decimal_value(balance):,f} {coin.SHORTNAME}')
+
+
+def main():
+    default_limit = 10
+    parser = argparse.ArgumentParser(
+        'query.py',
+        description='Invoke with COIN and DB_DIRECTORY set in the '
+        'environment as they would be invoking electrumx_server'
+    )
+    parser.add_argument('-l', '--limit', metavar='limit', type=int,
+                        default=10, help=f'maximum number of entries to '
+                        f'return (default: {default_limit})')
+    parser.add_argument('scripts', nargs='*', default=[], type=str,
+                        help='hex scripts to query')
+    args = parser.parse_args()
+    asyncio.run(query(args))
+
+if __name__ == '__main__':
+    main()

contrib/raspberrypi3/install_electrumx.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+###################
+# install electrumx
+###################
+
+# Remove "raspi-copies-and-fills" as it breaks the upgrade process
+sudo apt-get purge raspi-copies-and-fills
+
+# upgrade raspbian to 'stretch' distribution
+sudo echo 'deb http://mirrordirector.raspbian.org/raspbian/ testing main contrib non-free rpi' > /etc/apt/sources.list.d/stretch.list
+sudo apt-get update
+sudo apt-get dist-upgrade
+sudo apt-get autoremove
+
+# install electrumx dependencies
+sudo apt-get install python3-pip
+sudo apt-get install build-essential libc6-dev
+sudo apt-get install libncurses5-dev libncursesw5-dev
+sudo apt install libreadline6-dev/stable libreadline6/stable
+sudo apt-get install libleveldb-dev
+sudo apt-get install git
+sudo pip3 install plyvel
+
+# install electrumx
+git clone https://github.com/spesmilo/electrumx.git
+cd electrumx
+sudo python3 -m pip install .

contrib/raspberrypi3/run_electrumx.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+###############
+# run_electrumx
+###############
+
+# configure electrumx
+export COIN=Bitcoin
+export DAEMON_URL=http://rpcuser:rpcpassword@127.0.0.1
+export NET=mainnet
+export CACHE_MB=400
+export DB_DIRECTORY=/home/username/.electrumx/db
+export SSL_CERTFILE=/home/username/.electrumx/certfile.crt
+export SSL_KEYFILE=/home/username/.electrumx/keyfile.key
+export BANNER_FILE=/home/username/.electrumx/banner
+export DONATION_ADDRESS=your-donation-address
+
+# connectivity
+export HOST=
+export TCP_PORT=50001
+export SSL_PORT=50002
+
+# visibility
+export REPORT_HOST=hostname.com
+export RPC_PORT=8000
+
+# run electrumx
+ulimit -n 10000
+/usr/local/bin/electrumx_server 2>> /home/username/.electrumx/electrumx.log >> /home/username/.electrumx/electrumx.log &
+
+######################
+# auto-start electrumx
+######################
+
+# add this line to crontab -e
+# @reboot /path/to/run_electrumx.sh

contrib/sdist/Dockerfile

@@ -0,0 +1,30 @@
+FROM debian:bookworm@sha256:b877a1a3fdf02469440f1768cf69c9771338a875b7add5e80c45b756c92ac20a
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q && \
+    apt-get install -qy \
+        git \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        python3-build \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get autoremove -y && \
+    apt-get clean
+
+# create new user to avoid using root; but with sudo access and no password for convenience.
+ARG UID=1000
+ENV USER="user"
+ENV HOME_DIR="/home/${USER}"
+ENV WORK_DIR="${HOME_DIR}/wspace" \
+    PATH="${HOME_DIR}/.local/bin:${PATH}"
+RUN useradd --uid $UID --create-home --shell /bin/bash ${USER}
+RUN usermod -append --groups sudo ${USER}
+RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+WORKDIR ${WORK_DIR}
+RUN chown --recursive ${USER} ${WORK_DIR}
+USER ${USER}

contrib/sdist/README.md

@@ -0,0 +1,29 @@
+# Source tarballs
+
+✓ _These tarballs should be reproducible, meaning you should be able to generate
+   distributables that match the official releases._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system.
+
+
+## Build steps
+
+1. Install Docker
+
+    See [`docker_notes.md`](https://github.com/spesmilo/electrum/blob/master/contrib/docker_notes.md).
+
+    (worth reading even if you already have docker)
+
+2. Build tarball
+
+    ```
+    $ ./contrib/sdist/build.sh
+    ```
+    If you want reproducibility, try instead e.g.:
+    ```
+    $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./contrib/sdist/build.sh
+    ```
+
+3. The generated distributables are in `./dist`.
+

contrib/sdist/build.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+# env vars:
+# - ELECBUILD_NOCACHE: if set, forces rebuild of docker image
+# - ELECBUILD_COMMIT: if set, do a fresh clone and git checkout
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../.."
+PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT"
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+BUILD_UID=$(/usr/bin/stat -c %u "$PROJECT_ROOT")
+
+export SOURCE_DATE_EPOCH=1530212462
+export PYTHONHASHSEED=22
+
+
+DOCKER_BUILD_FLAGS=""
+if [ ! -z "$ELECBUILD_NOCACHE" ] ; then
+    echo "ELECBUILD_NOCACHE is set. forcing rebuild of docker image."
+    DOCKER_BUILD_FLAGS="--pull --no-cache"
+fi
+
+if [ -z "$ELECBUILD_COMMIT" ] ; then  # local dev build
+    DOCKER_BUILD_FLAGS="$DOCKER_BUILD_FLAGS --build-arg UID=$BUILD_UID"
+fi
+
+echo "building docker image."
+docker build \
+    $DOCKER_BUILD_FLAGS \
+    -t electrumx-builder-img \
+    "$CONTRIB_SDIST"
+
+# maybe do fresh clone
+if [ ! -z "$ELECBUILD_COMMIT" ] ; then
+    echo "ELECBUILD_COMMIT=$ELECBUILD_COMMIT. doing fresh clone and git checkout."
+    FRESH_CLONE="/tmp/electrumx_build/fresh_clone/electrumx"
+    rm -rf "$FRESH_CLONE" 2>/dev/null || ( echo "we need sudo to rm prev FRESH_CLONE." && sudo rm -rf "$FRESH_CLONE" )
+    umask 0022
+    git clone "$PROJECT_ROOT" "$FRESH_CLONE"
+    cd "$FRESH_CLONE"
+    git checkout "$ELECBUILD_COMMIT"
+    PROJECT_ROOT_OR_FRESHCLONE_ROOT="$FRESH_CLONE"
+else
+    echo "not doing fresh clone."
+fi
+
+echo "building binary..."
+# check uid and maybe chown. see #8261
+if [ ! -z "$ELECBUILD_COMMIT" ] ; then  # fresh clone (reproducible build)
+    if [ $(id -u) != "1000" ] || [ $(id -g) != "1000" ] ; then
+        echo "need to chown -R FRESH_CLONE dir. prompting for sudo."
+        sudo chown -R 1000:1000 "$FRESH_CLONE"
+    fi
+fi
+docker run -it \
+    --name electrumx-builder-cont \
+    -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrumx \
+    --rm \
+    --workdir /opt/electrumx/contrib/sdist \
+    electrumx-builder-img \
+    ./make_sdist.sh
+
+# make sure resulting binary location is independent of fresh_clone
+if [ ! -z "$ELECBUILD_COMMIT" ] ; then
+    mkdir --parents "$DISTDIR/"
+    cp -f "$FRESH_CLONE/dist"/* "$DISTDIR/"
+fi

contrib/sdist/make_sdist.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/sdist"
+BUILDDIR="$CONTRIB_SDIST/build"
+DISTDIR="$PROJECT_ROOT/dist"
+
+cd "$PROJECT_ROOT"
+
+rm -rf "$BUILDDIR"
+mkdir -p "$BUILDDIR" "$DISTDIR"
+
+
+git_status=$(git status --porcelain)
+if [ ! -z "$git_status" ]; then
+    echo "$git_status"
+    echo "git repo not clean, aborting"
+    exit 1
+fi
+
+
+cd "$PROJECT_ROOT"
+python3 -m build --sdist . --outdir "$BUILDDIR/dist1"
+
+
+# the initial tar.gz is not reproducible, see https://github.com/pypa/setuptools/issues/2133
+# so we untar, fix timestamps, and then re-tar
+DISTNAME=$(find "$BUILDDIR/dist1/" -type f -name 'e_x-*.tar.gz' -printf "%f\\n")
+DIST_BASENAME=$(basename --suffix ".tar.gz" "$DISTNAME")
+mkdir -p "$BUILDDIR/dist2"
+cd "$BUILDDIR/dist2"
+tar -xzf "$BUILDDIR/dist1/$DISTNAME"
+find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+GZIP=-n tar --sort=name -czf "$DISTNAME" "$DIST_BASENAME/"
+mv "$DISTNAME" "$DISTDIR/$DISTNAME"
+

contrib/systemd/electrumx.conf

@@ -0,0 +1,11 @@
+# default /etc/electrumx.conf for systemd
+
+# REQUIRED
+DB_DIRECTORY = /db
+# Bitcoin Node RPC Credentials
+DAEMON_URL = http://username:password@hostname:port/
+
+# COIN = Bitcoin
+
+# See https://electrumx-spesmilo.readthedocs.io/en/latest/environment.html for
+# information about other configuration settings you probably want to consider.

contrib/systemd/electrumx.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Electrumx
+After=network.target
+
+[Service]
+EnvironmentFile=/etc/electrumx.conf
+ExecStart=/usr/local/bin/electrumx_server
+ExecStop=/usr/local/bin/electrumx_rpc -p 8000 stop
+User=electrumx
+LimitNOFILE=8192
+TimeoutStopSec=30min
+
+[Install]
+WantedBy=multi-user.target

docs/ACKNOWLEDGEMENTS

@@ -0,0 +1,22 @@
+Thanks to Thomas Voegtlin for creating the Electrum software and
+infrastructure and for maintaining it so diligently.  Electrum is
+probably the best desktop Bitcoin wallet solution for most users.  My
+faith in it is such that I use Electrum software to store most of my
+Bitcoins.
+
+Whilst the vast majority of the code here is my own original work and
+includes some new ideas, it is very clear that the general structure
+and concept are those of Electrum.  Some parts of the code and ideas
+of Electrum, some of which it itself took from other projects such as
+Abe and pywallet, remain.  Thanks to the authors of all the software
+this is derived from.
+
+I'd like to thank bauerj, hsmiths and JWU42 of #electrum for their
+help and support in testing the versions of ElectrumX prior to 1.0.
+
+Thanks to Daniel Bernstein for daemontools and other software, and to
+Matthew Dillon for DragonFlyBSD.  They are both deeply inspirational
+people.
+
+And of course, thanks to Satoshi for the wonderful creation that is
+Bitcoin.

docs/HOWTO.rst

@@ -0,0 +1,461 @@
+.. _HOWTO:
+
+=====
+HOWTO
+=====
+
+Prerequisites
+=============
+
+**ElectrumX** should run on any flavour of unix.  I have run it
+successfully on MacOS and DragonFlyBSD.  It won't run out-of-the-box
+on Windows, but the changes required to make it do so should be
+small - pull requests are welcome.
+
+================ ========================
+Package          Notes
+================ ========================
+Python3          ElectrumX uses asyncio.  Python version >= 3.10 is
+                 **required**.
+`aiohttp`_       Python library for asynchronous HTTP.  Version >=
+                 2.0 required.
+DB Engine        A database engine package is required; two are
+                 supported (see `Database Engine`_ below).
+================ ========================
+
+Some coins need an additional package, typically for their block hash
+functions. For example, `dash_hash`_ is required for DASH. Scrypt coins
+require a Python interpreter compiled and/or linked with OpenSSL 1.1.0
+or higher.
+
+You **must** be running a non-pruning bitcoin daemon with::
+
+  txindex=1
+
+set in its configuration file.  If you have an existing installation
+of bitcoind and have not previously set this you will need to reindex
+the blockchain with::
+
+  bitcoind -reindex
+
+which can take some time.
+
+While not a requirement for running ElectrumX, it is intended to be
+run with supervisor software such as Daniel Bernstein's
+`daemontools`_, Gerrit Pape's `runit`_ package or :command:`systemd`.
+These make administration of secure unix servers very easy, and I
+strongly recommend you install one of these and familiarise yourself
+with them.  The instructions below and sample run scripts assume
+``daemontools``; adapting to ``runit`` should be trivial for someone
+used to either.
+
+When building the database from the genesis block, ElectrumX has to
+flush large quantities of data to disk and its DB.  You will have a
+better experience if the database directory is on an SSD than on an
+HDD.  Currently to around height 611,600 of the Bitcoin blockchain the
+final size of the leveldb database, and other ElectrumX file metadata
+comes to just over 46.9GB (43.7 GiB).  LevelDB needs a bit more for
+brief periods, and the block chain is only getting longer, so I would
+recommend having at least 70-80GB of free space before starting.
+
+Database Engine
+===============
+
+You can choose from LevelDB and RocksDB to store transaction
+information on disk.  On an SSD, RocksDB performs better than LevelDB.
+We tried to support LMDB, but its history write performance
+was much worse.
+
+You will need to install one of:
+
++ `plyvel <https://plyvel.readthedocs.io/en/latest/installation.html>`_ for LevelDB.
+
+  ``apt install libleveldb-dev build-essential``
+
+  ``pip3 install plyvel`` , or use the ``[leveldb]`` extra install option to ElectrumX.
+
++ `rocksdb-ng <https://pypi.org/project/rocksdb-ng>`_ for RocksDB
+
+  (Should work with RocksDB versions 8-10)
+
+  ``apt install librocksdb-dev build-essential pkg-config``
+
+  ``pip3 install rocksdb-ng`` , or use the ``[rocksdb]`` extra install option to ElectrumX.
+
++ `python-rocksdb <https://pypi.python.org/pypi/python-rocksdb>`_ for an unmaintained version
+  that doesn't work with recent releases of RocksDB (only <7.0)
+
+Running
+=======
+
+Install the prerequisites above.
+
+Check out the code from Github::
+
+    git clone https://github.com/spesmilo/electrumx.git
+    cd electrumx
+
+You can install with::
+
+    pip3 install .
+
+There are many extra Python dependencies available to fit the needs of your
+system or coins. For example, to install the RocksDB dependencies and a faster
+JSON (de)serialization library::
+
+    pip3 install ".[rocksdb,orjson]"
+
+see pyproject.toml's ``project.optional-dependencies`` for a complete list.
+
+You can also run the code from the source tree or a copy of it.
+
+
+You should create a standard user account to run the server under;
+your own is probably adequate unless paranoid.  The paranoid might
+also want to create another user account for the daemontools logging
+process.  The sample scripts and these instructions assume it is all
+under one account which I have called ``electrumx``.
+
+Next create a directory where the database will be stored and make it
+writeable by the ``electrumx`` account.  I recommend this directory
+live on an SSD::
+
+    mkdir /path/to/db_directory
+    chown electrumx /path/to/db_directory
+
+
+Process limits
+--------------
+
+The ElectrumX process needs a large open file limit. On Linux systems,
+the default (:command:`ulimit -n`) (soft) open file limit is usually 1,024.
+ElectrumX tries to raise this automatically during startup to the hard limit,
+which is usually 100,000+. If that fails (which would get logged),
+you might have to manually increase the limit.
+
+During sync it should not need more than about 1,024 open files.  When
+serving it will use approximately 256 for LevelDB plus the number of
+incoming connections.  It is not unusual to have 1,000 to 2,000
+connections being served, so I suggest you set your open files limit
+to at least 2,500.
+
+Note that setting the limit in your shell does *NOT* affect ElectrumX
+unless you are invoking ElectrumX directly from your shell.  If you
+are using :command:`systemd`, you need to set it in the
+:file:`.service` file (see `contrib/systemd/electrumx.service`_).
+
+
+Using daemontools
+-----------------
+
+Next create a daemontools service directory; this only holds symlinks
+(see daemontools documentation).  The :command:`svscan` program will
+ensure the servers in the directory are running by launching a
+:command:`supervise` supervisor for the server and another for its
+logging process.  You can run :command:`svscan` under the *electrumx*
+account if that is the only one involved (server and logger) otherwise
+it will need to run as root so that the user can be switched to
+electrumx.
+
+Assuming this directory is called :file:`service`, you would do one
+of::
+
+    mkdir /service       # If running svscan as root
+    mkdir ~/service      # As electrumx if running svscan as that a/c
+
+Next create a directory to hold the scripts that the
+:command:`supervise` process spawned by :command:`svscan` will run -
+this directory must be readable by the :command:`svscan` process.
+Suppose this directory is called :file:`scripts`, you might do::
+
+    mkdir -p ~/scripts/electrumx
+
+Then copy the all sample scripts from the ElectrumX source tree there::
+
+    cp -R /path/to/repo/electrumx/contrib/daemontools ~/scripts/electrumx
+
+This copies 3 things: the top level server run script, a :file:`log/`
+directory with the logger :command:`run` script, an :file:`env/`
+directory.
+
+You need to configure the :ref:`environment variables <environment>`
+under :file:`env/` to your setup.  ElectrumX server currently takes no
+command line arguments; all of its configuration is taken from its
+environment which is set up according to :file:`env/` directory (see
+:manpage:`envdir` man page).  Finally you need to change the
+:command:`log/run` script to use the directory where you want the logs
+to be written by multilog.  The directory need not exist as
+:command:`multilog` will create it, but its parent directory must
+exist.
+
+Now start the :command:`svscan` process.  This will not do much as the
+service directory is still empty::
+
+    svscan ~/service & disown
+
+svscan is now waiting for services to be added to the directory::
+
+    cd ~/service
+    ln -s ~/scripts/electrumx electrumx
+
+Creating the symlink will kick off the server process almost immediately.
+You can see its logs with::
+
+    tail -F /path/to/log/dir/current | tai64nlocal
+
+
+Using systemd
+-------------
+
+This repository contains a sample systemd unit file that you can use
+to setup ElectrumX with systemd. Simply copy it to
+:file:`/etc/systemd/system`::
+
+    cp contrib/systemd/electrumx.service /etc/systemd/system/
+
+The sample unit file assumes that the repository is located at
+:file:`/home/electrumx/electrumx`. If that differs on your system, you
+need to change the unit file accordingly.
+
+You need to set a few :ref:`environment variables <environment>` in
+:file:`/etc/electrumx.conf`.
+
+Now you can start ElectrumX using :command:`systemctl`::
+
+    systemctl start electrumx
+
+You can use :command:`journalctl` to check the log output::
+
+    journalctl -u electrumx -f
+
+Once configured you may want to start ElectrumX at boot::
+
+    systemctl enable electrumx
+
+.. Warning:: systemd is aggressive in forcibly shutting down
+   processes.  Depending on your hardware, ElectrumX can need several
+   minutes to flush cached data to disk during initial sync.  You
+   should set TimeoutStopSec to *at least* 10 mins in your
+   :file:`.service` file.
+
+
+Installing on Raspberry Pi 3
+----------------------------
+
+To install on the Raspberry Pi 3 you will need to update to the
+``stretch`` distribution.  See the full procedure in
+`contrib/raspberrypi3/install_electrumx.sh`_.
+
+See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to
+configure and launch electrumx.
+
+
+Sync Progress
+=============
+
+Time taken to index the blockchain depends on your hardware of course.
+As Python is single-threaded most of the time only 1 core is kept
+busy.  ElectrumX uses Python's :mod:`asyncio` to prefill a cache of
+future blocks asynchronously to keep the CPU busy processing the chain
+without pausing.
+
+Consequently there will probably be only a minor boost in performance
+if the daemon is on the same host.  It may even be beneficial to have
+the daemon on a *separate* machine so the machine doing the indexing
+has its caches and disk I/O tuned to that task only.
+
+The :envvar:`CACHE_MB` environment variable controls the total cache
+size ElectrumX uses; see :ref:`here <CACHE>` for caveats.
+
+Here is my experience with the codebase of early 2017 (the current
+codebase is faster), to given heights and rough wall-time.  The period
+from heights 363,000 to 378,000 is the most sluggish::
+
+                 Machine A     Machine B
+  181,000          25m 00s      5m 30s
+  283,500                       1h 00m
+  321,800                       1h 40m
+  357,000          12h 32m      2h 41m
+  386,000          21h 56m      4h 25m
+  414,200       1d 12h 29m      6h 30m
+  447,168       2d 13h 20m      9h 47m
+
+*Machine A*: a low-spec 2011 1.6GHz AMD E-350 dual-core fanless CPU,
+8GB RAM and a DragonFlyBSD UFS filesystem on an SSD.  It requests
+blocks over the LAN from a bitcoind on machine B.  :envvar:`DB_CACHE`
+the default of 1,200.  LevelDB.
+
+*Machine B*: a late 2012 iMac running Sierra 10.12.2, 2.9GHz quad-core
+Intel i5 CPU with an HDD and 24GB RAM.  Running bitcoind on the same
+machine.  :envvar:`DB_CACHE` set to 1,800.  LevelDB.
+
+For chains other than bitcoin-mainnet synchronization should be much
+faster.
+
+.. note:: ElectrumX will not serve normal client connections until it
+          has fully synchronized and caught up with your daemon.
+          However LocalRPC connections are served at all times.
+
+
+Terminating ElectrumX
+=====================
+
+The preferred way to terminate the server process is to send it the
+``stop`` RPC command::
+
+  electrumx_rpc stop
+
+or alternatively on Unix the ``INT`` or ``TERM`` signals.  For a
+daemontools supervised process this can be done by bringing it down
+like so::
+
+    svc -d ~/service/electrumx
+
+ElectrumX will note receipt of the signals in the logs, and ensure the
+block chain index is flushed to disk before terminating.  You should
+be patient as flushing data to disk can take many minutes.
+
+ElectrumX uses the transaction functionality, with fsync enabled, of
+the databases.  I have written it with the intent that, to the extent
+the atomicity guarantees are upheld by the DB software, the operating
+system, and the hardware, the database should not get corrupted even
+if the ElectrumX process if forcibly killed or there is loss of power.
+The worst case should be having to restart indexing from the most
+recent UTXO flush.
+
+Once the process has terminated, you can start it up again with::
+
+    svc -u ~/service/electrumx
+
+You can see the status of a running service with::
+
+    svstat ~/service/electrumx
+
+:command:`svscan` can of course handle multiple services
+simultaneously from the same service directory, such as a testnet or
+altcoin server.  See the man pages of these various commands for more
+information.
+
+
+Understanding the Logs
+======================
+
+You can see the logs usefully like so::
+
+    tail -F /path/to/log/dir/current | tai64nlocal
+
+Here is typical log output on startup::
+
+  INFO:BlockProcessor:switching current directory to /crucial/server-good
+  INFO:BlockProcessor:using leveldb for DB backend
+  INFO:BlockProcessor:created new database
+  INFO:BlockProcessor:creating metadata diretcory
+  INFO:BlockProcessor:software version: ElectrumX 0.10.2
+  INFO:BlockProcessor:DB version: 5
+  INFO:BlockProcessor:coin: Bitcoin
+  INFO:BlockProcessor:network: mainnet
+  INFO:BlockProcessor:height: -1
+  INFO:BlockProcessor:tip: 0000000000000000000000000000000000000000000000000000000000000000
+  INFO:BlockProcessor:tx count: 0
+  INFO:BlockProcessor:sync time so far: 0d 00h 00m 00s
+  INFO:BlockProcessor:reorg limit is 200 blocks
+  INFO:Daemon:daemon at 192.168.0.2:8332/
+  INFO:BlockProcessor:flushing DB cache at 1,200 MB
+  INFO:Controller:RPC server listening on localhost:8000
+  INFO:Prefetcher:catching up to daemon height 447,187...
+  INFO:Prefetcher:verified genesis block with hash 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+  INFO:BlockProcessor:our height: 9 daemon: 447,187 UTXOs 0MB hist 0MB
+  INFO:BlockProcessor:our height: 52,509 daemon: 447,187 UTXOs 9MB hist 14MB
+  INFO:BlockProcessor:our height: 85,009 daemon: 447,187 UTXOs 12MB hist 31MB
+  INFO:BlockProcessor:our height: 102,384 daemon: 447,187 UTXOs 15MB hist 47MB
+  [...]
+  INFO:BlockProcessor:our height: 133,375 daemon: 447,187 UTXOs 80MB hist 222MB
+  INFO:BlockProcessor:our height: 134,692 daemon: 447,187 UTXOs 96MB hist 250MB
+  INFO:BlockProcessor:flushed to FS in 0.7s
+  INFO:BlockProcessor:flushed history in 16.3s for 1,124,512 addrs
+  INFO:BlockProcessor:flush #1 took 18.7s.  Height 134,692 txs: 941,963
+  INFO:BlockProcessor:tx/sec since genesis: 2,399, since last flush: 2,400
+  INFO:BlockProcessor:sync time: 0d 00h 06m 32s  ETA: 1d 13h 03m 42s
+
+Under normal operation these cache stats repeat once or twice a
+minute.  UTXO flushes can take several minutes and look like this::
+
+  INFO:BlockProcessor:our height: 378,745 daemon: 447,332 UTXOs 1,013MB hist 184MB
+  INFO:BlockProcessor:our height: 378,787 daemon: 447,332 UTXOs 1,014MB hist 194MB
+  INFO:BlockProcessor:flushed to FS in 0.3s
+  INFO:BlockProcessor:flushed history in 13.4s for 934,933 addrs
+  INFO:BlockProcessor:flushed 6,403 blocks with 5,879,440 txs, 2,920,524 UTXO adds, 3,646,572 spends in 93.1s, committing...
+  INFO:BlockProcessor:flush #120 took 226.4s.  Height 378,787 txs: 87,695,588
+  INFO:BlockProcessor:tx/sec since genesis: 1,280, since last flush: 359
+  INFO:BlockProcessor:sync t ime: 0d 19h 01m 06s  ETA: 3d 21h 17m 52s
+  INFO:BlockProcessor:our height: 378,812 daemon: 447,334 UTXOs 10MB hist 10MB
+
+The ETA shown is just a rough guide and in the short term can be quite
+volatile.  It tends to be a little optimistic at first; once you get
+to height 280,000 is should be fairly accurate.
+
+.. _SSL certificates:
+
+Creating a self-signed SSL certificate
+======================================
+
+These instructions are based on those of the ``electrum-server``
+documentation.
+
+To run an SSL server you need to generate a self-signed certificate
+using openssl.  Alternatively you could not set :envvar:`SSL_PORT` in
+the environment and not serve over SSL, but this is not recommended.
+
+Use the sample code below to create a self-signed cert with a
+recommended validity of 5 years. You may supply any information for
+your sign request to identify your server.  They are not currently
+checked by the client except for the validity date.  When asked for a
+challenge password just leave it empty and press enter::
+
+    $ openssl genrsa -out server.key 2048
+    $ openssl req -new -key server.key -out server.csr
+    ...
+    Country Name (2 letter code) [AU]:US
+    State or Province Name (full name) [Some-State]:California
+    Common Name (eg, YOUR name) []: electrum-server.tld
+    ...
+    A challenge password []:
+    ...
+    $ openssl x509 -req -days 1825 -in server.csr -signkey server.key -out server.crt
+
+The :file:`server.crt` file goes in :envvar:`SSL_CERTFILE` and
+:file:`server.key` in :envvar:`SSL_KEYFILE` in the server process's
+environment.
+
+Starting with Electrum 1.9, the client will learn and locally cache
+the SSL certificate for your server upon the first request to prevent
+man-in-the middle attacks for all further connections.
+
+If your certificate is lost or expires on the server side, you will
+need to run your server with a different server name and a new
+certificate.  Therefore it's a good idea to make an offline backup
+copy of your certificate and key in case you need to restore them.
+
+Running on a privileged port
+============================
+
+You may choose to run electrumx on a different port than 50001
+/ 50002.  If you choose a privileged port ( < 1024 ) it makes sense to
+make use of a iptables NAT rule.
+
+An example, which will forward Port 110 to the internal port 50002 follows::
+
+    iptables -t nat -A PREROUTING -p tcp --dport 110 -j DNAT --to-destination 127.0.0.1:50002
+
+You can then set the port as follows and advertise the service externally on the privileged port::
+
+    REPORT_SSL_PORT=110
+
+
+.. _`contrib/systemd/electrumx.service`: https://github.com/spesmilo/electrumx/blob/master/contrib/systemd/electrumx.service
+.. _`daemontools`: http://cr.yp.to/daemontools.html
+.. _`runit`: http://smarden.org/runit/index.html
+.. _`aiohttp`: https://pypi.python.org/pypi/aiohttp
+.. _`dash_hash`: https://pypi.python.org/pypi/dash_hash
+.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/spesmilo/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh
+.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/spesmilo/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    = -W
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = ElectrumX
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/PERFORMANCE-NOTES

@@ -0,0 +1,45 @@
+Just some notes on performance with Python 3.5. We are taking these into
+account in the code.
+
+- 60% faster to create lists with [] list comprehensions than tuples
+  or lists with tuple(), list().  Of those list is 10% faster than
+  tuple.
+
+- however when not initializing from a generator, a fixed-length tuple
+  is at least 80% faster than a list.
+
+- an implicit default argument is ~5% faster than passing the default
+  explicitly
+
+- using a local variable x rather than self.x in loops and list
+  comprehensions is over 50% faster
+
+- struct.pack, struct.unpack are over 60% faster than int.to_bytes and
+  int.from_bytes.  They are faster little endian (presumably because
+  it matches the host) than big endian regardless of length. Furthermore,
+  using stored packing and unpacking methods from Struct classes is faster
+  than using the flexible-format struct.[un]pack equivalents.
+
+  After storing the Struct('<Q').unpack_from function as unpack_uint64_from,
+   later calls to unpack_uint64_from(b, 0) are about 30% faster than calls to
+   unpack_from('<Q', b, 0).
+
+
+- single-item list and tuple unpacking.  Suppose b = (1, )
+
+   a, = b  is a about 0.4% faster than  (a,) = b
+           and about 45% faster than a = b[0]
+
+- multiple assignment is faster using tuples only for 3 or more items
+
+- retrieving a previously stored length of a bytes object can be over 200%
+  faster than a new call to len(b)
+
+- The cpython and pypy stdlibs use a Python-based JSON serializer/deserializer,
+  which is stable and easy to use and install, but somewhat slower than
+  system-compiled variants. ElectrumX now looks for known popular
+  C/Rust-based JSON libraries for Python. If it finds one,
+  it's used instead of the stdlib module for improved performance.
+
+  To install a known-compatible system-compiled JSON lib, install ElectrumX
+  with the "orjson" extras parameter.

docs/architecture.rst

@@ -0,0 +1,83 @@
+Architecture
+============
+
+.. image:: https://docs.google.com/drawings/d/1Su_DR2c8__-4phm12hAzV65fL2tNm_1IhKr4XivkW6Q/pub?w=720&h=540
+    :target: https://docs.google.com/drawings/d/1Su_DR2c8__-4phm12hAzV65fL2tNm_1IhKr4XivkW6Q/pub?w=960&h=720
+
+Env
+---
+
+Holds configuration taken from the environment, with appropriate
+defaulting appropriately.  Generally passed to the constructor of
+other components which take their settings from it.
+
+Controller
+----------
+
+The central part of the server process initialising and coordinating
+all the others.  Manages resource usage.
+
+
+LocalRPC
+--------
+
+Handles local JSON RPC connections querying ElectrumX server state.
+Started when the ElectrumX process starts.
+
+ElectrumX
+---------
+
+Handles JSON Electrum client connections over TCP or SSL.  One
+instance per client session.  Should be the only component concerned
+with the details of the Electrum wire protocol.
+
+Not started until the Block Processor has caught up with bitcoind.
+
+Daemon
+------
+
+Encapsulates the RPC wire protocol with bitcoind for the whole server.
+Transparently handles temporary bitcoind connection errors, and fails
+over if necessary.
+
+Notifies the Mempool when the list of mempool transaction hashes is
+updated.
+
+
+Block Processor
+---------------
+
+Responsible for managing block chain state (UTXO set, history,
+transaction and undo information) and for handling block chain
+reorganisations.
+
+When caught up, processes new blocks as they are found, and flushes
+the updates to the Database immediately.
+
+When syncing uses caches for in-memory state updates since the prior
+flush.  Occasionally flushes state to the storage layer when caches
+get large.
+
+Prefetcher
+----------
+
+Cooperates with the Block Processor to asynchronously prefetch blocks
+from bitcoind.  Once it has caught up it additionally asks the Daemon
+to refresh its view of bitcoind's mempool transaction hashes.  Serves
+blocks to the Block Processor via a queue.
+
+Mempool
+-------
+
+Handles all the details of maintaining a representation of bitcoind's
+mempool state.  Obtains the list of current mempool transaction hashes
+from the Daemon when notified by the Prefetcher.
+
+Notifies the Controller that addresses have been touched when the
+mempool refreshes (or implicitly when a new block is found).
+
+Database
+--------
+
+The underlying data store, made up of the DB backend (such as
+`leveldb`) and the host filesystem.

docs/authors.rst

@@ -0,0 +1,20 @@
+.. _Authors:
+
+Authors
+=======
+
+* Neil Booth
+
+  Original creator, past maintainer.
+
+* Electrum developers
+
+  Current maintainers.
+
+* Johann Bauer
+
+  Backend DB abstraction.
+
+* John Jegutanis
+
+  Alt-chain integrations.

docs/changelog.rst

@@ -0,0 +1,431 @@
+===========
+ ChangeLog
+===========
+
+Version 2.0.0 (03 July 2026)
+==============================
+
+TL;DR:
+
+- new DB schema, so you need to rebuild from scratch. No more "DB::flush_count overflow".
+- Bitcoin Core >=31 is now required, with `txospenderindex=1`
+- added support for Electrum Protocol 1.7
+- running with free-threaded CPython is now (experimentally) supported, and is much faster
+
+-----
+
+* Breaking changes to on-disk database. Server operators need to manually delete old DB,
+  no migration path. New DB will be rebuilt while rescanning again from genesis.
+
+   - in return, no more manual "history compaction" and unexpected downtimes every year:
+     no more "DB::flush_count overflow" (`spesmilo/electrumx#88`_)
+   - the size of the new DB is comparable to the old one, however:
+
+* We now require Bitcoin Core to have :code:`txospenderindex=1` (added in Bitcoin Core 31)
+  in addition to :code:`txindex=1`.  This is needed to serve the :code:`blockchain.outpoint.subscribe` RPC
+  added in Electrum Protocol 1.7.
+
+  For reference, on Bitcoin mainnet around height=950k,
+
+   - the ElectrumX db uses around 122 GiB (using LevelDB, roughly same for both of e-x 1.x and 2.0),
+      - note: using RocksDB 9, it is a bit smaller, around 106 GiB
+   - :code:`.bitcoin/blocks/` uses 788 GiB,
+   - :code:`.bitcoin/chainstate/` uses 12 GiB,
+   - :code:`.bitcoin/indexes/txindex/` uses 66 GiB,
+   - :code:`.bitcoin/indexes/txospenderindex/` uses 88 GiB (new!)
+
+  - This also means that we now require Bitcoin Core 31.0 or newer (for :code:`COIN=Bitcoin`).
+
+* env: the previously optional envvar :code:`DB_ENGINE`, is now mandatory.
+
+  - choose either :code:`leveldb` or :code:`rocksdb`
+  - in ElectrumX 1.x versions, the default was leveldb.
+
+  You need to install the appropriate dependencies for your engine, see the :code:`[leveldb]` and
+  :code:`[rocksdb]` pip extras and the "Database Engine" section of the HOWTO.
+
+  Previously RocksDB was difficult to use as the python bindings for it have been unmaintained for years.
+  By the super-slop-powers of LLMs, we revived the python bindings as :code:`rocksdb-ng` and made it compatible with modern
+  cpython, cython, and rocksdb. The changes are reviewable and not really slop. (see `spesmilo/electrumx#347`_)
+  rocksdb-ng (starting with version 2.3) is also compatible with `free-threaded python`_, see "performance" section.
+
+  LevelDB was written with HDDs in mind. RocksDB is more modern
+  and on an SSD takes around ~25% less time than LevelDB to sync from genesis.
+  Given that this release contains breaking DB changes and a resync is needed anyway, maybe try out RocksDB :)
+
+* performance:
+
+   - the BlockProcessor is now massively multi-threaded, both inside a single block and across a span of blocks.
+     This can mainly only be taken advantage of if the GIL (Global Interpreter Lock) is disabled, that is,
+     when using a `free-threaded python`_ interpreter: e.g. cpython :code:`3.14t`, compiled with :code:`--disable-gil`.
+     On a decent modern machine with a multi-core CPU, this has been measured to cut initial sync time
+     from genesis to a bit less than half. OTOH, much more RAM is required for the parallel processing
+     (example: approx 2 GB when syncing with GIL, 8 GB when syncing without GIL).
+
+   - processing a single block at the tip, when already caught up, is expected to take around half the time,
+     regardless of the GIL. Still, without the GIL, it is even faster.
+
+   - for more details, see `spesmilo/electrumx#369`_
+
+   - the (block)Prefetcher and the Mempool code are now partially event-based (but still also polling):
+     e.g. a mempool update is triggered as soon as processing a new block begins (and part of the work
+     is done concurrently)
+
+   - json (de)serialization can now optionally use the Rust-based
+     :code:`orjson` library (via the :code:`orjson` pip extra), replacing the
+     previous :code:`ujson` and :code:`rapidjson` extras.
+
+* Dockerfiles: the existing Dockerfile was updated and cleaned up a bit, and it serves
+  as a simple minimal example to show how to run ElectrumX. There is now also a second Dockerfile
+  that sets up a free-threaded python for a beefy server to utilise the above-mentioned performance gains.
+  See them in :code:`contrib/`. (also see the new runtime logline "Python GIL enabled" to verify effect)
+
+* protocol:
+   - new: implement electrum protocol version 1.7  (`spesmilo/electrum-protocol#2`_).
+     The min supported protocol version remains 1.4, the max is now 1.7.
+
+* LocalRPC: new command: "inspect_session"  (`spesmilo/electrumx#361`_).
+
+
+Version 1.20.0 (03 June 2026)
+=============================
+
+Small fixes and general maintenance.
+
+After this release, the master branch will move towards a 2.0 release, which will require
+a resync from genesis. So for a while, master might be less stable than usual.
+
+* fix: lib/tx.py: vsize calculation was off by 6-8 vbytes (`spesmilo/electrumx#328`_).
+  Added tests. The discrepancy was externally observable via the `mempool.get_fee_histogram` RPC.
+* fix: don't get stuck during shutdown waiting for SessionManager to stop (`spesmilo/electrumx#339`_)
+* fix: protocol: name of arg for broadcast_package RPC: tx_package->raw_txs (`c9b72e7e`_)
+* changed: peer discovery: allow short chain splits (`spesmilo/electrumx#351`_)
+* changed: rm 'attrs' as a direct dependency, use stdlib `dataclasses` instead (`spesmilo/electrumx#345`_)
+* new: daemon: check early "txindex" is enabled for bitcoind, error if missing (`spesmilo/electrumx#349`_)
+* new: coins: add Bitcoin mutinynet (`spesmilo/electrumx#352`_)
+* maintenance: coins: some clean-up (`spesmilo/electrumx#344`_, `spesmilo/electrumx#348`_, ...)
+* maintenance: moved CI from Cirrus CI to Github Actions (`spesmilo/electrumx#340`_)
+
+(We also made some changes to make RocksDB practical and nicer to use, but if you currently use LevelDB,
+maybe consider waiting for 2.0 before you resync the DB)
+
+
+Version 1.19.0 (11 Nov 2025)
+=============================
+
+* protocol:
+   - new: implement electrum protocol version 1.6 (`spesmilo/electrumx#317`_, ...)
+     (`spesmilo/electrum-protocol#6`_).
+     The min supported protocol version remains 1.4, the max is now 1.6.
+* coins:
+   - changed: for `COIN=Bitcoin`, there is now a min required bitcoind version:
+     Bitcoin Core 28.0 (or Knots 28) (`spesmilo/electrumx#316`_).
+     This is due to protocol 1.6 requiring a bitcoind with working `submitpackage` RPC.
+* security:
+   - fix DOS vector: if MAX_SESSIONS was reached, when using python 3.12+, the server
+     stopped accepting new incoming sessions until full-restart (`spesmilo/electrumx#312`_)
+* session:
+   - changed: add warmup budget to PaddedRSTransport (`spesmilo/electrumx#323`_):
+     this is a small relaxation of the traffic analysis countermeasures added in 1.18,
+     where the first 1024 bytes we send are now exempt from buffering (and hence delays)
+   - fix ReplyAndDisconnect for PaddedRSTransport: flush message buffer (`spesmilo/electrumx#322`_)
+* env:
+   - rm DROP_CLIENT_UNKNOWN opt-in env var: this behaviour is now mandated by protocol 1.6
+     and we just always enforce it (~small breaking change, affected clients would have
+     already been considered misbehaving)
+* misc:
+   - new: try to raise `ulimit -n` open file limit automatically at startup
+     (`spesmilo/electrumx#326`_)
+
+
+Version 1.18.0 (14 June 2025)
+=============================
+
+* protocol:
+   - add basic countermeasures against traffic analysis by padding the jsonrpc payload
+     with whitespaces to have ~uniform-size TCP packets, and by artificially delaying
+     sending messages a bit. (`spesmilo/electrumx#301`_)
+* dependencies:
+   - bump required aiorpcx to >=0.25.0,<0.26
+
+
+Version 1.17.0 (16 Apr 2025)
+============================
+
+Long time no see!
+
+* coins:
+   - rm support for legacy coin name "BitcoinSegwit" (`ca59151d`_)
+     (users should just change their config files to :code:`COIN=Bitcoin`)
+   - add support for Bitcoin Signet (`spesmilo/electrumx#122`_)
+   - add support for Bitcoin Testnet4 (`spesmilo/electrumx#273`_)
+* dependencies:
+   - bump required python to >=3.10
+   - bump required aiorpcx to >=0.23.0,<0.25
+   - rm "pylru" dep, instead bundle stripped-down "cachetools" (`spesmilo/electrumx#248`_)
+* session: log IP address of excessive resusage sessions (`spesmilo/electrumx#75`_)
+* robustness:
+   - session: more robust session handling (`25339328`_, `8198ab99`_)
+   - daemon: catch RPC_PARSE_ERROR from bitcoind and simply retry (`31489365`_)
+* env:
+   - add MAX_RECV environment variable (`a0d3053f`_)
+   - increase default for MAX_SEND (`2969ab11`_)
+* protocol: increase resolution of :code:`mempool.get_fee_histogram` (`265a5a87`_)
+* LocalRPC:
+   - add --timeout arg (`5f4dd2cd`_)
+   - add commands to help debug memory leaks (`f5582b29`_)
+* build/maintenance:
+   - add scripts for reproducible build (`0ba87447`_)
+   - migrate from setup.py to pyproject.toml
+   - refactor file layout flat->src (`spesmilo/electrumx#298`_)
+* some internal refactors in preparation for future changes
+
+
+Version 1.16.0 (10 Dec 2020)
+============================
+
+Note: this is the first release since forking from kyuupichan/electrumx.
+kyuupichan has the :code:`electrumx` name on PyPI, so we needed a new name there.
+We are using the `e-x <https://pypi.org/project/e-x/>`_ name on PyPI, so you can
+install this package via e.g. :code:`pip install e-x`.
+
+* security: a vulnerability has been fixed that allowed a remote attacker to
+  crash electrumx if peer discovery was enabled (`#22`_)
+* fixed some peer-discovery-related bugs (e.g. `#35`_)
+* ENV: when using Bitcoin, the COIN ENV var can now be set to :code:`Bitcoin`.
+  For compatibility, using :code:`BitcoinSegwit` will also keep working.
+  (`#5`_)
+* session resource limits: made more useful in general. connection-time-based
+  grouping has been removed (`#70`_). Disconnects of over-limit sessions happen
+  sooner (`4b3f6510`_). Subnet sizes for IP-based grouping can now be
+  configured (`a61136c5`_).
+* protocol: :code:`mempool.get_fee_histogram` now returns fee rates with
+  0.1 sat/byte resolution instead of 1 sat/byte, and the compact histogram
+  is calculated differently. (`#67`_)
+* performance: bitcoind is now queried less frequently as estimatefee,
+  relayfee, and server.banner requests are now cached (`#24`_)
+* performance: json ser/deser is now abstracted away and the :code:`ujson` and
+  :code:`rapidjson` exras can be used for somewhat faster block processing.
+  (`#11`_)
+* multiple other small performance improvements
+
+
+Version 1.15.0 (27 May 2020)
+============================
+
+* switch to 5-byte txnums to handle larger blockchains.  Upgrade DBs during restart.
+* accurate clearing of stale caches
+* coin additions / updates: NavCoin + Hush + VersusCoin + Zero (cipig), DashRegtest (colmenero),
+  Quebecoin (morinpa), Primecoin (Sunny King), multiple (Panagiotis David), RVN (standard-error),
+  Sumcoin
+* other: Jeremy Rand, Jin Eguchi, ddude, Jonathan Cross, Carsen Klock, cipig
+
+
+Version 1.14.0 (19 Jan 2020)
+============================
+
+* require Python 3.7
+* support for Bitcoin SV Genesis activation
+* DB upgrade to allow for larger transactions.  Your DB will automatically upgrade when
+  starting, the upgrade should take approximately 15 mintues.
+* fix server shutdown process
+* fix cache race condition (issue `#909`_)
+* faster initial sync
+* coin additions / updates: Emercoin (yakimka), Feathercoin (wellenreiter01),
+  Peercoin (peerchemist), Namecoin (JeremyRand), Zcoin (a-bezrukov), Simplicity,
+  Mice (ComputerCraftr), Sibcoin testnet (TriKriSta), Odin (Manbearpixel),
+* other: h2o10, osagga, Sombernight, breign, pedr0-fr, wingsuit
+
+Version 1.13.0 (26 Sep 2019)
+============================
+
+* daemon: use a single connection for all requests rather than a connection per request.
+  Distinguish handling of JSON and HTTP errors
+* recognise OP_FALSE OP_RETURN scripts as unspendable
+* peers - attempt to bind to correct local IP address
+* improve name support (domob1812)
+* coin additions / updates: BitZeny (y-chan), ZCoin (a-bezrukov), Emercoin (yakimka),
+  BSV (Roger Taylor), Bellcoin (streetcrypto7), Ritocoin (traysi), BTC (Sombernight),
+  PIVX (mrcarlanthony), Monacoin (wakiyamap)), NamecoinRegtest (JeremyRand), Axe (ddude1),
+  Xaya (domob1812), GZRO (MrNaif2018), Ravencoin (standard-error)
+* other: gits7r
+
+Version 1.12.0 (13 May 2019)
+============================
+
+* require aiorpcX 0.18.1.  This introduces websocket support.  The environment variables
+  changed accordingly; see :envvar:`SERVICES` and :envvar:`REPORT_SERVICES`.
+* work around bug in recent versions of uvloop
+* aiorpcX upgrade fixes from Shane M
+* coin additions / updates: BitcoinSV, Bolivarcoin (Jose Luis Estevez), BTC Testnet (ghost43),
+  Odin (Pixxl)
+
+Version 1.11.0 (18 Apr 2019)
+============================
+
+* require aiorpcX 0.15.x
+* require aiohttp 3.3 or higher; earlier versions had a problematic bug
+* add :envvar:`REQUEST_TIMEOUT` and :envvar:`LOG_LEVEL` environment variables
+* mark 4 old environment variables obsolete.  ElectrumX won't start until they are removed
+* getinfo local RPC cleaned up and shows more stats
+* miscellaneous fixes and improvements
+* more efficient handling of some RPC methods, particularly
+  :func:`blockchain.transaction.get_merkle`
+* coin additions / updates: BitcoinSV scaling testnet (Roger Taylor), Dash (zebra lucky),
+* issues resolved: `#566`_, `#731`_, `#795`_
+
+Version 1.10.1 (13 Apr 2019)
+============================
+
+* introduce per-request costing.  See environment variables documentation for new
+  variables :envvar:`COST_SOFT_LIMIT`, :envvar:`COST_HARD_LIMIT`, :envvar:`REQUEST_SLEEP`,
+  :envvar:`INITIAL_CONCURRENT`, :envvar:`BANDWIDTH_UNIT_COST`.  Sessions are placed in groups
+  with which they share some of their costs.  Prior cost is remembered across reconnects.
+* require aiorpcX 0.13.5 for better concurrency handling
+* require clients use protocol 1.4 or higher
+* handle transaction.get_merkle requests more efficiently (ghost43)
+* Windows support (sancoder)
+* peers improvements (ghost43)
+* report mempool and block sizes in logs
+* electrumx_rpc: timeout raised to 30s, fix session request counts
+* other tweaks and improvements by Bjorge Dijkstra, ghost43, peleion,
+* coin additions / updates: ECA (Jenova7), ECCoin (smogm), GXX (DEVCØN), BZX (2INFINITY),
+  DeepOnion (Liam Alford), CivX / EXOS (turcol)
+
+Version 1.10.0 (15 Mar 2019)
+============================
+
+* extra countermeasures to limit BTC phishing effectiveness (ghost43)
+* peers: mark blacklisted peers bad; force retry blacklisted peers (ghost43)
+* coin additions / updates: Monacoin (wakiyamap), Sparks (Mircea Rila), ColossusXT,
+  Polis, MNPCoin, Zcoin, GINCoin (cronos), Grosetlcoin (gruve-p), Dash (konez2k),
+  Bitsend (David), Ravencoin (standard-error), Onixcoin (Jose Estevez), SnowGem
+* coin removals: Gobyte, Moneci (cronos)
+* minor tweaks by d42
+* issues fixed `#660`_ - unclean shutdowns during initial sync
+
+Version 1.9.5 (08 Feb 2019)
+===========================
+
+* server blacklist logic (ecdsa)
+* require aiorpcX 0.10.4
+* remove dead wallet code
+* fix `#727`_ - not listing same peer twice
+
+Version 1.9.4 (07 Feb 2019)
+===========================
+
+* require aiorpcX 0.10.3
+* fix `#713`_
+
+Version 1.9.3 (05 Feb 2019)
+===========================
+
+* ignore potential sybil peers
+* coin additions / updates: BitcoinCashABC (cculianu), Monacoin (wakiyamap)
+
+Version 1.9.2 (03 Feb 2019)
+===========================
+
+* restore protocol version 1.2 and send a warning for old BTC Electrum clients that they
+  need to upgrade.  This is an attempt to protect users of old versions of Electrum from
+  the ongoing phishing attacks
+* increase default MAX_SEND for AuxPow Chains.  Truncate AuxPow for block heights covered
+  by a checkpoint.  (jeremyrand)
+* coin additions / updates: NMC (jeremyrand), Dash (zebra-lucky), PeerCoin (peerchemist),
+  BCH testnet (Mark Lundeberg), Unitus (ChekaZ)
+* tighter RPC param checking (ghost43)
+
+Version 1.9.1 (11 Jan 2019)
+===========================
+
+* fix `#684`_
+
+Version 1.9.0 (10 Jan 2019)
+===========================
+
+* minimum protocol version is now 1.4
+* coin additions / updates: BitcoinSV, SmartCash (rc125), NIX (phamels), Minexcoin (joesixpack),
+  BitcoinABC (mblunderburg), Dash (zebra-lucky), BitcoinABCRegtest (ezegom), AXE (slowdive),
+  NOR (flo071), BitcoinPlus (bushsolo), Myriadcoin (cryptapus), Trezarcoin (ChekaZ),
+  Bitcoin Diamond (John Shine),
+* close `#554`_, `#653`_, `#655`_
+* other minor tweaks (Michael Schmoock, Michael Taborsky)
+
+
+Original author of ElectrumX:
+
+**Neil Booth**  kyuupichan@gmail.com  https://github.com/kyuupichan
+
+This fork maintained by:
+
+**Electrum developers** electrumdev@gmail.com  https://github.com/spesmilo
+
+
+.. _#554: https://github.com/kyuupichan/electrumx/issues/554
+.. _#566: https://github.com/kyuupichan/electrumx/issues/566
+.. _#653: https://github.com/kyuupichan/electrumx/issues/653
+.. _#655: https://github.com/kyuupichan/electrumx/issues/655
+.. _#660: https://github.com/kyuupichan/electrumx/issues/660
+.. _#684: https://github.com/kyuupichan/electrumx/issues/684
+.. _#713: https://github.com/kyuupichan/electrumx/issues/713
+.. _#727: https://github.com/kyuupichan/electrumx/issues/727
+.. _#731: https://github.com/kyuupichan/electrumx/issues/731
+.. _#795: https://github.com/kyuupichan/electrumx/issues/795
+.. _#909: https://github.com/kyuupichan/electrumx/issues/909
+
+
+.. _#5:   https://github.com/spesmilo/electrumx/pull/5
+.. _#11:  https://github.com/spesmilo/electrumx/pull/11
+.. _#22:  https://github.com/spesmilo/electrumx/issues/22
+.. _#24:  https://github.com/spesmilo/electrumx/pull/24
+.. _#35:  https://github.com/spesmilo/electrumx/pull/35
+.. _#67:  https://github.com/spesmilo/electrumx/pull/67
+.. _#70:  https://github.com/spesmilo/electrumx/pull/70
+.. _spesmilo/electrumx#75:  https://github.com/spesmilo/electrumx/pull/75
+.. _spesmilo/electrumx#88:  https://github.com/spesmilo/electrumx/issues/88
+.. _spesmilo/electrumx#122:  https://github.com/spesmilo/electrumx/pull/122
+.. _spesmilo/electrumx#248:  https://github.com/spesmilo/electrumx/pull/248
+.. _spesmilo/electrumx#273:  https://github.com/spesmilo/electrumx/pull/273
+.. _spesmilo/electrumx#298:  https://github.com/spesmilo/electrumx/pull/298
+.. _spesmilo/electrumx#301:  https://github.com/spesmilo/electrumx/pull/301
+.. _spesmilo/electrumx#312:  https://github.com/spesmilo/electrumx/pull/312
+.. _spesmilo/electrumx#316:  https://github.com/spesmilo/electrumx/pull/316
+.. _spesmilo/electrumx#317:  https://github.com/spesmilo/electrumx/pull/317
+.. _spesmilo/electrumx#322:  https://github.com/spesmilo/electrumx/pull/322
+.. _spesmilo/electrumx#323:  https://github.com/spesmilo/electrumx/pull/323
+.. _spesmilo/electrumx#326:  https://github.com/spesmilo/electrumx/pull/326
+.. _spesmilo/electrumx#328:  https://github.com/spesmilo/electrumx/pull/328
+.. _spesmilo/electrumx#339:  https://github.com/spesmilo/electrumx/pull/339
+.. _spesmilo/electrumx#340:  https://github.com/spesmilo/electrumx/pull/340
+.. _spesmilo/electrumx#344:  https://github.com/spesmilo/electrumx/pull/344
+.. _spesmilo/electrumx#345:  https://github.com/spesmilo/electrumx/pull/345
+.. _spesmilo/electrumx#347:  https://github.com/spesmilo/electrumx/pull/347
+.. _spesmilo/electrumx#348:  https://github.com/spesmilo/electrumx/pull/348
+.. _spesmilo/electrumx#349:  https://github.com/spesmilo/electrumx/pull/349
+.. _spesmilo/electrumx#351:  https://github.com/spesmilo/electrumx/pull/351
+.. _spesmilo/electrumx#352:  https://github.com/spesmilo/electrumx/pull/352
+.. _spesmilo/electrumx#361:  https://github.com/spesmilo/electrumx/pull/361
+.. _spesmilo/electrumx#369:  https://github.com/spesmilo/electrumx/pull/369
+
+
+.. _4b3f6510:  https://github.com/spesmilo/electrumx/commit/4b3f6510e94670a013c1abe6247cdd2b0e7e6f8c
+.. _a61136c5:  https://github.com/spesmilo/electrumx/commit/a61136c596d6a0290a6be9d21fb7c095c3cea21e
+.. _ca59151d:  https://github.com/spesmilo/electrumx/commit/ca59151d7365aabd2394fb39aac1faa25949b49d
+.. _25339328:  https://github.com/spesmilo/electrumx/commit/25339328a7468235071d152f728b214df10d4c56
+.. _8198ab99:  https://github.com/spesmilo/electrumx/commit/8198ab99603a7e74083b3569c15dad13850c721e
+.. _31489365:  https://github.com/spesmilo/electrumx/commit/314893655a7cc0bfcc216b07900fa77b5f66e148
+.. _a0d3053f:  https://github.com/spesmilo/electrumx/commit/a0d3053f60074264b54b3ef7583f2600f4b73ead
+.. _2969ab11:  https://github.com/spesmilo/electrumx/commit/2969ab110412bebddc3f3e815467fb59538b613d
+.. _265a5a87:  https://github.com/spesmilo/electrumx/commit/265a5a87b8ad01f739049c0b1e80923aab318f58
+.. _5f4dd2cd:  https://github.com/spesmilo/electrumx/commit/5f4dd2cdb414464484407affbbaae6b7407696cb
+.. _f5582b29:  https://github.com/spesmilo/electrumx/commit/f5582b29792625e8cca7cf137a6718c2520bb9cb
+.. _0ba87447:  https://github.com/spesmilo/electrumx/commit/0ba87447cb293cfc4a8a26c1c27842b95666875a
+.. _c9b72e7e:  https://github.com/spesmilo/electrumx/commit/c9b72e7ea76bfb706b2df3e90c739116bf49678d
+
+
+.. _spesmilo/electrum-protocol#2:  https://github.com/spesmilo/electrum-protocol/pull/2
+.. _spesmilo/electrum-protocol#6:  https://github.com/spesmilo/electrum-protocol/pull/6
+
+.. _free-threaded python:  https://docs.python.org/3/howto/free-threading-python.html
+

docs/conf.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/stable/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+VERSION="ElectrumX 2.0.0"
+
+# -- Project information -----------------------------------------------------
+
+project = 'ElectrumX'
+copyright = '2016-2026, various'
+author = 'Electrum developers'
+
+# The full version including branding
+release = VERSION
+# The short X.Y version
+version = VERSION.split()[-1]
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme_options = {
+    'description': 'Lightweight Electrum Server in Python',
+    'github_user': 'spesmilo',
+    'github_repo': 'electrumx',
+    'github_button': True,
+    'github_type': 'star',
+    'github_banner': True,
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+
+html_sidebars = {
+    '**': [
+        'about.html', 'navigation.html', 'searchbox.html',
+    ]
+}
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ElectrumXdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'ElectrumX.tex', 'ElectrumX Documentation',
+     'Neil Booth', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'electrumx', 'ElectrumX Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'ElectrumX', 'ElectrumX Documentation',
+     author, 'ElectrumX', 'One line description of project.',
+     'Miscellaneous'),
+]

docs/environment.rst

@@ -0,0 +1,510 @@
+.. _environment:
+
+=====================
+Environment Variables
+=====================
+
+ElectrumX takes no command line arguments, instead its behaviour is controlled by
+environment variables.  Only a few are required to be given, the rest will have sensible
+defaults if not specified.  Many of the defaults around resource usage are conservative; I
+encourage you to review them.
+
+.. note:: set :envvar:`SERVICES` appropriately to be able to connect to your server.  For
+  clients across the internet to know what services you offer you must advertize your
+  services with :envvar:`REPORT_SERVICES`.
+
+
+Required
+========
+
+These environment variables are always required:
+
+.. envvar:: DB_DIRECTORY
+
+  The path to the database directory.  Relative paths should be
+  relative to the parent process working directory.  This is the
+  directory of the `run` script if you use it.
+
+.. envvar:: DB_ENGINE
+
+  Database engine for the UTXO and history database.
+  Choose one of ``leveldb`` or ``rocksdb``.
+  You will need to install the appropriate python package for your engine.
+  In ElectrumX 1.x versions, the default was leveldb.
+  Warning: It is not possible to switch back and forth,
+  the on-disk DB formats are not compatible with each other: you have to resync from genesis.
+  LevelDB was written with HDDs in mind. RocksDB is more modern
+  and on an SSD takes around ~25% less time than LevelDB to sync from genesis.
+
+.. envvar:: DAEMON_URL
+
+  A comma-separated list of daemon URLs.  If more than one is provided
+  ElectrumX will initially connect to the first, and failover to
+  subsequent ones round-robin style if one stops working.
+
+  The generic form of a daemon URL is::
+
+     http://username:password@hostname:port/
+
+  The leading ``http://`` is optional, as is the trailing slash.  The
+  ``:port`` part is also optional and will default to the standard RPC
+  port for :envvar:`COIN` and :envvar:`NET` if omitted.
+
+
+.. note:: With the above set your server will run and index the chain.  To enable incoming
+   connections you must set :envvar:`SERVICES`, and for others to be aware of your server
+   set :envvar:`REPORT_SERVICES`.
+
+
+For the ``run`` script
+======================
+
+The following are required if you use the ``run`` script:
+
+.. envvar:: ELECTRUMX
+
+  The path to the electrumx_server script.  Relative paths should
+  be relative to the directory of the ``run`` script.
+
+.. envvar:: USERNAME
+
+  The username the server will run as.
+
+
+Services
+========
+
+These two environment variables are comma-separated lists of individual *services*.
+
+A **service** has the general form::
+
+  protocol://host:port
+
+*protocol* is case-insensitive.  The recognised protocols are::
+
+   tcp    Plaintext TCP sockets
+   ssl    SSL-encrypted TCP sockets
+   ws     Plaintext websockets
+   wss    SSL-encrypted websockets
+   rpc    Plaintext RPC
+
+In a services list, a protocol can be specified multiple times, with different hosts or
+ports.  This might be useful for multi-homed hosts, or if you offer both Tor and clearnet
+services.
+
+*host* can be a hostname, an IPv4 address, or an IPv6 address enclosed in square brackets.
+
+*port* is an integer from :const:`1` to :const:`65535` inclusive.
+
+Where documented, one or more of *protocol*, *host* and *port* can be omitted, in which
+case a default value will be assumed.
+
+Here are some examples of valid services::
+
+  tcp://host.domain.tld:50001           # Hostname, lowercase protocol, port
+  SSL://23.45.67.78:50002               # An IPv4 address, upper-case protocol, port
+  rpC://localhost                       # Host as a string, mixed-case protocol, default port
+  ws://[1234:5678:abcd::5601]:8000      # Host as an IPv6 address
+  wss://h3ubaasdlkheryasd.onion:50001   # Host as a Tor ".onion" address
+  rpc://:8000                           # Default host, port given
+  host.domain.tld:5151                  # Default protocol, hostname, port
+  rpc://                                # RPC protocol, default host and port
+
+.. note:: ElectrumX will not serve any incoming connections until it has fully caught up
+          with your bitcoin daemon.  The only exception is local **RPC** connections,
+          which are served at any time after the server has initialized.
+
+.. envvar:: SERVICES
+
+  A comma-separated list of services ElectrumX will accept incoming connections for.
+
+  This environment variable determines what interfaces and ports the server listens on, so
+  must be set correctly for any connection to the server to succeed.  If unset or empty,
+  ElectrumX will not listen for any incoming connections.
+
+  *protocol* can be any recognised protocol.
+
+  *host* defaults to all of the machine's interfaces, except if the protocol is **rpc**,
+  when it defaults to :const:`localhost`.
+
+  *port* can only be defaulted for **rpc** where the default is :const:`8000`.
+
+  On most Unix systems ports below 1024 require elevated privileges so choosing a higher
+  port is advisable.  On Debian for example, this can be achieved by installing
+  libcap2-bin package::
+
+    sudo apt-get update && sudo apt-get -y install libcap2-bin
+    sudo setcap cap_net_bind_service=+ep /path/to/electrumx_server
+
+  If any listed service has protocol **ssl** or **wss** then :envvar:`SSL_CERTFILE` and
+  :envvar:`SSL_KEYFILE` must be defined.
+
+  Tor **onion** addresses are invalid in :envvar:`SERVICES`.
+
+  Here is an example value of the :envvar:`SERVICES` environment variable::
+
+    tcp://:50001,ssl://:50002,wss://:50004,rpc://
+
+  This serves **tcp**, **ssl**, **wss** on all interfaces on ports 50001, 50002 and 50004
+  respectively.  **rpc** is served on its default host :const:`localhost` and default port
+  :const:`8000`.
+
+.. envvar:: REPORT_SERVICES
+
+  A comma-separated list of services ElectrumX will advertize and other servers in the
+  server network (if peer discovery is enabled), and any successful connection.
+
+  This environment variable must be set correctly, taking account of your network,
+  firewall and router setup, for clients and other servers to see how to connect to your
+  server.  If not set or empty, no services are advertized.
+
+  The **rpc** protocol, special IP addresses (including private ones if peer discovery is
+  enabled), and :const:`localhost` are invalid in :envvar:`REPORT_SERVICES`.
+
+  Here is an example value of the :envvar:`REPORT_SERVICES` environment variable::
+
+    tcp://example.com:50001,ssl://example.com:50002,wss://example.com:50004
+
+  This advertizes **tcp**, **ssl**, **wss** services at :const:`example.com` on ports
+  50001, 50002 and 50004 respectively.
+
+.. note:: Certificate Authority-signed certificates don't work over Tor, so you should
+          only have Tor services` in :envvar:`REPORT_SERVICES` if yours is self-signed.
+
+.. envvar:: SSL_CERTFILE
+
+  The filesystem path to your SSL certificate file.
+
+  :ref:`SSL certificates`
+
+.. envvar:: SSL_KEYFILE
+
+  The filesystem path to your SSL key file.
+
+  :ref:`SSL certificates`
+
+
+Miscellaneous
+=============
+
+These environment variables are optional:
+
+.. envvar:: COIN
+
+  Must be a :attr:`NAME` from one of the :class:`Coin` classes in
+  `lib/coins.py`_.  Defaults to ``Bitcoin``.
+
+.. envvar:: NET
+
+  Must be a :attr:`NET` from one of the :class:`Coin` classes in
+  `lib/coins.py`_.  Defaults to ``mainnet``.
+
+.. envvar:: LOG_FORMAT
+
+  The Python logging `format string
+  <https://docs.python.org/3/library/logging.html#logrecord-attributes>`_
+  to use.  Defaults to ``%(levelname)s:%(name)s:%(message)s``.
+
+.. envvar:: LOG_LEVEL
+
+  The default Python logging level, a case-insensitive string.  Useful values
+  are 'debug', 'info', 'warning' and 'error'.
+
+.. envvar:: ALLOW_ROOT
+
+  Set this environment variable to anything non-empty to allow running
+  ElectrumX as root.
+
+.. envvar:: DONATION_ADDRESS
+
+  The server donation address reported to Electrum clients.  Defaults
+  to empty, which Electrum interprets as meaning there is none.
+
+.. envvar:: BANNER_FILE
+
+  The path to a banner file to serve to clients in Electrum's
+  "Console" tab.  Relative file paths must be relative to
+  :envvar:`DB_DIRECTORY`.  The banner file is re-read for each new
+  client.
+
+  You can place several meta-variables in your banner file, which will be
+  replaced before serving to a client.
+
+  + ``$SERVER_VERSION`` is replaced with the ElectrumX version you are
+    running, such as ``1.0.10``.
+  + ``$SERVER_SUBVERSION`` is replaced with the ElectrumX user agent
+    string.  For example, ``ElectrumX 1.0.10``.
+  + ``$DAEMON_VERSION`` is replaced with the daemon's version as a
+    dot-separated string. For example ``0.12.1``.
+  + ``$DAEMON_SUBVERSION`` is replaced with the daemon's user agent
+    string.  For example, ``/BitcoinUnlimited:0.12.1(EB16; AD4)/``.
+  + ``$DONATION_ADDRESS`` is replaced with the address from the
+    :envvar:`DONATION_ADDRESS` environment variable.
+
+  See `here <https://github.com/shsmith/electrumx-banner-updater>`_
+  for a script that updates a banner file periodically with useful
+  statistics about fees, last block time and height, etc.
+
+.. envvar:: TOR_BANNER_FILE
+
+  As for :envvar:`BANNER_FILE` (which is also the default) but shown
+  to incoming connections believed to be to your Tor hidden service.
+
+.. envvar:: ANON_LOGS
+
+  Set to anything non-empty to replace IP addresses in logs with
+  redacted text like ``xx.xx.xx.xx:xxx``.  By default IP addresses
+  will be written to logs.
+
+.. envvar:: LOG_SESSIONS
+
+  The number of seconds between printing session statistics to the
+  log.  The output is identical to the :ref:`sessions` RPC command
+  except that :envvar:`ANON_LOGS` is honoured.  Defaults to 3600.  Set
+  to zero to suppress this logging.
+
+.. envvar:: REORG_LIMIT
+
+  The maximum number of blocks to be able to handle in a chain
+  reorganisation.  ElectrumX retains some fairly compact undo
+  information for this many blocks in levelDB.  The default is a
+  function of :envvar:`COIN` and :envvar:`NET`; for Bitcoin mainnet it
+  is 200.
+
+.. envvar:: EVENT_LOOP_POLICY
+
+  The name of an event loop policy to replace the default asyncio
+  policy, if any.  At present only ``uvloop`` is accepted, in which
+  case you must have installed the `uvloop`_ Python package.
+
+  If you are not sure what this means leave it unset.
+
+.. envvar:: DROP_CLIENT
+
+  Set a regular expression to disconnect any client based on their
+  version string. For example to drop versions from 1.0 to 1.2 use
+  the regex ``1\.[0-2]\.\d+``.
+
+
+Resource Usage Limits
+=====================
+
+The following environment variables are all optional and help to limit
+server resource consumption and prevent simple DoS.
+
+Address subscriptions in ElectrumX are very cheap - they consume about
+160 bytes of memory each and are processed efficiently.  I feel the
+two subscription-related defaults below are low and encourage you to
+raise them.
+
+.. envvar:: MAX_SESSIONS
+
+  The maximum number of incoming connections.  Once reached, TCP and
+  SSL listening sockets are closed until the session count drops
+  naturally to 95% of the limit.  Defaults to 1,000.
+
+.. envvar:: MAX_RECV
+
+  The maximum size of an incoming message in bytes, the default is 1,000,000 bytes.
+  Note that the smallest sane/safe value for Bitcoin is ~800,100 bytes,
+  as the largest standard tx can have a weight of 400K but the protocol hex-encodes that,
+  plus there is a few bytes of protocol overhead. Setting this to lower than that
+  would preclude clients from broadcasting txs that could propagate over the network.
+
+.. envvar:: MAX_SEND
+
+  The maximum size of a response message to send over the wire, in
+  bytes.  Defaults to 8,100,000 (except for AuxPoW coins, which default
+  to 10,000,000).  Values smaller than 350,000 are taken as 350,000
+  because standard Electrum protocol header "chunk" requests are almost
+  that large.
+
+  The Electrum protocol has a flaw in that address histories must be
+  served all at once or not at all, an obvious avenue for abuse.
+  :envvar:`MAX_SEND` is a stop-gap until the protocol is improved to
+  admit incremental history requests.  Each history entry is
+  approximately 100 bytes, so the default is equivalent to a history
+  limit of around 80,000 entries, which should be ample for most
+  legitimate users.  If you use a higher default, bear in mind one
+  client can request history for multiple addresses.
+
+  Also note that the largest raw transaction you will be able to serve to a client is
+  just under half of :envvar:`MAX_SEND`, as each raw byte becomes 2 hexadecimal
+  ASCII characters on the wire.  For Bitcoin, the consensus limit for tx size is
+  4 M weight units, while the standardness (policy) limit is 400 K weight units.
+  When serialized as bytes, 4 M weight units is 4 MB, and when encoded as hex that is 8 MB.
+  The default was chosen so that such a tx could still be served.
+  (Note: in multi-party protocol, such as Lightning, a counterparty might collude with a miner
+  to get a tx mined. Being able to obtain such a raw tx might be part of the security
+  requirements of a protocol.)
+
+.. envvar:: COST_SOFT_LIMIT
+.. envvar:: COST_HARD_LIMIT
+.. envvar:: REQUEST_SLEEP
+.. envvar:: INITIAL_CONCURRENT
+.. envvar:: SESSION_GROUP_BY_SUBNET_IPV4
+.. envvar:: SESSION_GROUP_BY_SUBNET_IPV6
+
+  All values are integers. :envvar:`COST_SOFT_LIMIT` defaults to :const:`1,000`,
+  :envvar:`COST_HARD_LIMIT` to :const:`10,000`, :envvar:`REQUEST_SLEEP` to :const:`2,500`
+  milliseconds, and :envvar:`INITIAL_CONCURRENT` to :const:`10` concurrent requests.
+
+  The server prices each request made to it based upon an estimate of the resources needed
+  to process it.  Factors include whether the request uses bitcoind, how much bandwidth
+  it uses, and how hard it hits the databases.
+
+  To set a base for the units, a :func:`blockchain.scripthash.subscribe` subscription to
+  an address with a history of 2 or fewer transactions is costed at :const:`1.0` before
+  considering the bandwidth consumed.  :func:`server.ping` is costed at :const:`0.1`.
+
+  As the total cost of a session goes over the soft limit, its requests start to be
+  throttled in two ways.  First, the number of requests for that session that the server
+  will process concurrently is reduced.  Second, each request starts to sleep a little
+  before being handled.
+
+  Before throttling starts, the server will process up to :envvar:`INITIAL_CONCURRENT`
+  requests concurrently without sleeping.  As the session cost ranges from
+  :envvar:`COST_SOFT_LIMIT` to :envvar:`COST_HARD_LIMIT`, concurrency drops linearly to
+  zero and each request's sleep time increases linearly up to :envvar:`REQUEST_SLEEP`
+  milliseconds.  Once the hard limit is reached, the session is disconnected.
+
+  In order that non-abusive sessions can continue to be served, a session's cost gradually
+  decays over time.  Subscriptions have an ongoing servicing cost, so the decay is slower
+  as the number of subscriptions increases.
+
+  If a session disconnects, ElectrumX continues to associate its cost with its IP address,
+  so if it immediately reconnects it will re-acquire its previous cost allocation.
+  Moreover, sessions are also grouped together based on their IP address subnets, and cost
+  is accrued for the whole group. What subnet sizes to use can be configured via
+  :envvar:`SESSION_GROUP_BY_SUBNET_IPV4` (by default /24) and
+  :envvar:`SESSION_GROUP_BY_SUBNET_IPV6` (by default /48).
+
+  A server operator should experiment with different values according to server loads.  It
+  is not necessarily true that e.g. having a low soft limit, decreasing concurrency and
+  increasing sleep will help handling heavy loads, as it will also increase the backlog of
+  requests the server has to manage in memory.  It will also give a much worse experience
+  for genuine connections.
+
+.. envvar:: BANDWIDTH_UNIT_COST
+
+  The number of bytes, sent and received, by a session that is deemed to cost :const:`1.0`.
+
+  The default value :const:`5,000` bytes, meaning the bandwidth cost assigned to a response
+  of 100KB is 20.  If your bandwidth is cheap you should probably raise this.
+
+.. envvar:: REQUEST_TIMEOUT
+
+  An integer number of seconds defaulting to :const:`30`.  If a request takes longer than
+  this to respond to, either because of request limiting or because the request is
+  expensive, the server rejects it and returns a timeout error to the client indicating
+  that the server is busy.
+
+  This can help prevent large backlogs of unprocessed requests building up under heavy load.
+
+.. envvar:: SESSION_TIMEOUT
+
+  An integer number of seconds defaulting to :const:`600`.  Sessions that have not sent a
+  request for longer than this are disconnected.  Properly functioning clients should send
+  a :func:`server.ping` request once roughly 450 seconds have passed since the previous
+  request, in order to avoid disconnection.
+
+
+Peer Discovery
+==============
+
+In response to the :func:`server.peers.subscribe` RPC call, ElectrumX
+will only return peer servers that it has recently connected to and
+verified basic functionality.
+
+If you are not running a Tor proxy ElectrumX will be unable to connect
+to onion server peers, in which case rather than returning no onion
+peers it will fall back to a hard-coded list.
+
+To give incoming clients a full range of onion servers, you will need
+to be running a Tor proxy for ElectrumX to use.
+
+ElectrumX will perform peer-discovery by default and announce itself
+to other peers.  If your server is private you may wish to disable
+some of this.
+
+.. envvar:: PEER_DISCOVERY
+
+  This environment variable is case-insensitive and defaults to
+  ``on``.
+
+  If ``on``, ElectrumX will occasionally connect to and verify its
+  network of peer servers.
+
+  If ``off``, peer discovery is disabled and a hard-coded default list
+  of servers will be read in and served.  If set to ``self`` then peer
+  discovery is disabled and the server will only return itself in the
+  peers list.
+
+.. envvar:: PEER_ANNOUNCE
+
+  Set this environment variable to empty to disable announcing itself.
+  If not defined, or non-empty, ElectrumX will announce itself to
+  peers.
+
+  If peer discovery is disabled this environment variable has no
+  effect, because ElectrumX only announces itself to peers when doing
+  peer discovery if it notices it is not present in the peer's
+  returned list.
+
+.. envvar:: FORCE_PROXY
+
+  By default peer discovery happens over the clear internet.  Set this
+  to non-empty to force peer discovery to be done via the proxy.  This
+  might be useful if you are running a Tor service exclusively and
+  wish to keep your IP address private.
+
+.. envvar:: TOR_PROXY_HOST
+
+  The host where your Tor proxy is running.  Defaults to
+  ``localhost``.
+
+  If you are not running a Tor proxy just leave this environment
+  variable undefined.
+
+.. envvar:: TOR_PROXY_PORT
+
+  The port on which the Tor proxy is running.  If not set, ElectrumX
+  will autodetect any proxy running on the usual ports 9050 (Tor),
+  9150 (Tor browser bundle) and 1080 (socks).
+
+.. envvar:: BLACKLIST_URL
+
+  URL to retrieve a list of blacklisted peers.  If not set, a coin-
+  specific default is used.
+
+
+Cache
+=====
+
+If synchronizing from the Genesis block your performance might change
+by tweaking the cache size.  Cache size is only checked roughly every
+minute, so the cache can grow beyond the specified size.  Moreover,
+the Python process is often quite a bit fatter than the cache size,
+because of Python overhead and also because leveldb consumes a lot of
+memory when flushing.  So I recommend you do not set this over 60% of
+your available physical RAM:
+
+.. _CACHE:
+
+.. envvar:: CACHE_MB
+
+  The amount of cache, in MB, to use.  The default is 1,200.
+
+  A portion of the cache is reserved for unflushed history, which is
+  written out frequently.  The bulk is used to cache UTXOs.
+
+  Larger caches probably increase performance a little as there is
+  significant searching of the UTXO cache during indexing.  However, I
+  don't see much benefit in my tests pushing this too high, and in
+  fact performance begins to fall, probably because LevelDB already
+  caches, and also because of Python GC.
+
+  I do not recommend raising this above 2000.
+
+.. _lib/coins.py: https://github.com/spesmilo/electrumx/blob/master/src/electrumx/lib/coins.py
+.. _uvloop: https://pypi.python.org/pypi/uvloop

docs/features.rst

@@ -0,0 +1,59 @@
+Features
+========
+
+- Efficient, lightweight reimplementation of electrum-server
+- Fast synchronization of bitcoin mainnet from Genesis.  Recent
+  hardware should synchronize in well under 24 hours.  The fastest
+  time to height 448k (mid January 2017) reported is under 4h 30m.  On
+  the same hardware JElectrum would take around 4 days and
+  electrum-server probably around 1 month.
+- Various configurable means of controlling resource consumption and
+  handling bad clients and denial of service attacks.  These include
+  maximum connection counts, subscription limits per-connection and
+  across all connections, maximum response size, per-session bandwidth
+  limits, and session timeouts.
+- Minimal resource usage once caught up and serving clients; tracking the
+  transaction mempool appears to be the most expensive part.
+- Mostly asynchronous processing of new blocks, mempool updates, and
+  client requests.  Busy clients should not noticeably impede other
+  clients' requests and notifications, nor the processing of incoming
+  blocks and mempool updates.
+- Daemon failover.  More than one daemon can be specified, and
+  ElectrumX will failover round-robin style if the current one fails
+  for any reason.
+- Peer discovery protocol removes need for IRC
+- Coin abstraction makes compatible altcoin and testnet support easy.
+
+Implementation
+==============
+
+ElectrumX does not do any pruning or throwing away of history.  I want
+to retain this property for as long as it is feasible, and it appears
+efficiently achievable for the foreseeable future with plain Python.
+
+The following all play a part in making it efficient as a Python
+blockchain indexer:
+
+- aggressive caching and batching of DB writes
+- more compact and efficient representation of UTXOs, address index,
+  and history.  Electrum Server stores full transaction hash and
+  height for each UTXO, and does the same in its pruned history.  In
+  contrast ElectrumX just stores the transaction number in the linear
+  history of transactions.  ElectrumX can determine block
+  height from a simple binary search of tx counts stored on disk.
+  ElectrumX stores historical transaction hashes in a linear array on
+  disk.
+- placing static append-only metadata indexable by position on disk
+  rather than in levelDB.  It would be nice to do this for histories
+  but I cannot think of a way.
+- avoiding unnecessary or redundant computations, such as converting
+  address hashes to human-readable ASCII strings with expensive bignum
+  arithmetic, and then back again.
+- better choice of Python data structures giving lower memory usage as
+  well as faster traversal
+- leveraging asyncio for asynchronous prefetch of blocks to mostly
+  eliminate CPU idling.  As a Python program ElectrumX is unavoidably
+  single-threaded in its essence; we must keep that CPU core busy.
+
+Python's :mod:`asyncio` means ElectrumX has no (direct) use for threads
+and associated complications.

docs/index.rst

@@ -0,0 +1,71 @@
+=========
+ElectrumX
+=========
+
+.. image:: https://github.com/spesmilo/electrumx/actions/workflows/ci.yml/badge.svg?branch=master
+    :target: https://github.com/spesmilo/electrumx/actions/workflows/ci.yml
+
+This project is a fork of `kyuupichan/electrumx <https://github.com/kyuupichan/electrumx>`_.
+The original author dropped support for Bitcoin, which we intend to keep.
+
+ElectrumX allows users to run their own Electrum server. It connects to your
+full node and indexes the blockchain, allowing efficient querying of history of
+arbitrary addresses. The server can be exposed publicly, and joined to the public network
+of servers via peer discovery. As of May 2020, a significant chunk of the public
+Electrum server network runs ElectrumX.
+
+The current version is |release|.
+
+Source Code
+===========
+
+The project is hosted on `GitHub
+<https://github.com/spesmilo/electrumx/>`_.  and uses `GitHub Actions
+<https://github.com/spesmilo/electrumx/actions>`_ for Continuous
+Integration.
+
+Please submit an issue on the `bug tracker
+<https://github.com/spesmilo/electrumx/issues>`_ if you have found a
+bug or have a suggestion to improve the server.
+
+Authors and License
+===================
+
+Neil Booth wrote the vast majority of the code; see :ref:`Authors`.
+Python version at least 3.10 is required.
+
+The code is released under the `MIT Licence
+<https://github.com/spesmilo/electrumx/LICENCE>`_.
+
+Getting Started
+===============
+
+See :ref:`HOWTO`.
+
+There is also an `installer`_ available that simplifies the
+installation on various Linux-based distributions, and a `Dockerfile`_
+available.
+
+.. _installer: https://github.com/bauerj/electrumx-installer
+.. _Dockerfile: https://github.com/lukechilds/docker-electrumx
+
+Documentation
+=============
+
+.. toctree::
+
+   features
+   changelog
+   HOWTO
+   environment
+   protocol
+   peer_discovery
+   rpc-interface
+   architecture
+   authors
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`

docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=ElectrumX
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/peer_discovery.rst

@@ -0,0 +1,154 @@
+.. _Peer Discovery:
+
+Peer Discovery
+==============
+
+This was implemented in ElectrumX as of version 0.11.0.  Support for
+IRC peer discovery was removed in ElectrumX version 1.2.1.
+
+The :dfn:`peer database` is an in-memory store of peers with at least
+the following information about a peer, required for a response to the
+:func:`server.peers.subscribe` RPC call:
+
+* host name
+* ip address
+* TCP and SSL port numbers
+* protocol version
+* pruning limit, if any
+
+
+Hard-coded Peers
+----------------
+
+A list of hard-coded, well-known peers seeds the peer discovery
+process.  Ideally it should have at least 4 servers that have shown
+commitment to reliable service.
+
+In ElectrumX this is a per-coin property in `lib/coins.py
+<https://github.com/spesmilo/electrumx/blob/master/electrumx/lib/coins.py>`_.
+
+
+server.peers.subscribe
+----------------------
+
+:func:`server.peers.subscribe` is used by Electrum clients to get a
+list of peer servers, in preference to a hard-coded list of peer
+servers in the client, which it will fall back to if necessary.
+
+The server should craft its response in a way that reduces the
+effectiveness of server sybil attacks and peer spamming.
+
+The response should only include peers it has successfully connected
+to recently.  Only reporting recent good peers ensures that those that
+have gone offline will be forgotten quickly and not be passed around
+for long.
+
+In ElectrumX, "recently" is taken to be the last 24 hours.  Only one
+peer from each IPv4/16 netmask is returned, and the number of onion
+peers is limited.
+
+
+Maintaining the Peer Database
+-----------------------------
+
+In order to keep its peer database up-to-date and fresh, after some
+time has passed since the last successful connection to a peer, an
+Electrum server should make another attempt to connect, choosing
+either the TCP or SSL port.
+
+On connecting it should issue :func:`server.peers.subscribe`,
+:func:`blockchain.headers.subscribe`, and :func:`server.features` RPC
+calls to collect information about the server and its peers.  If the
+peer seems to not know of you, you can issue a :func:`server.add_peer`
+call to advertise yourself.  Once this is done and replies received,
+terminate the connection.
+
+The peer database should view information obtained from an outgoing
+connection as authoritative, and prefer it to information obtained
+from any other source.
+
+On connecting, a server should confirm the peer is serving the same
+network, ideally via the genesis block hash of the
+:func:`server.features` RPC call below.  Also the height reported by
+the peer should be within a small number of the expected value.  If a
+peer is on the wrong network it should never be advertised to clients
+or other peers.  Such invalid peers should perhaps be remembered for a
+short time to prevent redundant revalidation if other peers persist in
+advertising them, and later forgotten.
+
+If a connection attempt fails, subsequent reconnection attempts should
+follow some kind of exponential backoff.
+
+If a long period of time has elapsed since the last successful
+connection attempt, the peer entry should be removed from the
+database.  This ensures that all peers that have gone offline will
+eventually be forgotten by the network entirely.
+
+ElectrumX will connect to the SSL port if both ports are available.
+If that fails it will fall back to the TCP port.  It tries to
+reconnect to a good peer at least once every 24 hours, and a failing
+after 5 minutes but with exponential backoff.  It forgets a peer
+entirely if a few days have passed since a successful connection.
+ElectrumX attempts to connect to onion peers through a Tor proxy that
+can be configured or that it will try to autodetect.
+
+
+server.features
+---------------
+
+:func:`server.features` is a fairly new RPC call that a server can use
+to advertise what services and features it offers.  It is intended for
+use by Electrum clients as well as other peers.  Peers will use it to
+gather peer information from the peer itself.
+
+The call takes no arguments and returns a dictionary keyed by feature
+name whose value gives details about the feature where appropriate.
+If a key is missing the feature is presumed not to be offered.
+
+
+server.add_peer
+---------------
+
+:func:`server.add_peer` is intended for a new server to get itself in
+the connected set.
+
+A server receiving a :func:`server.add_peer` call should not replace
+existing information about the host(s) given, but instead schedule a
+separate connection to verify the information for itself.
+
+To prevent abuse a server may do nothing with second and subsequent
+calls to this method from a single connection.
+
+The result should be True if accepted and False otherwise.
+
+
+Notes for Implementors
+----------------------
+
+* it is very important to only accept peers that appear to be on the
+  same network.  At a minimum the genesis hash should be compared (if
+  the peer supports :func:`server.features`), and also that the peer's
+  reported height is within a few blocks of your own server's height.
+* care should be taken with the :func:`server.add_peer` call.
+  Consider only accepting it once per connection.  Clearnet peer
+  requests should check the peer resolves to the requesting IP
+  address, to prevent attackers from being able to trigger arbitrary
+  outgoing connections from your server.  This doesn't work for onion
+  peers so they should be rate-limited.
+* it should be possible for a peer to change their port assignments -
+  presumably connecting to the old ports to perform checks will not
+  work.
+* peer host names should be checked for validity before accepting
+  them; and `localhost` should probably be rejected.  If it is an IP
+  address it should be a normal public one (not private, multicast or
+  unspecified).
+* you should limit the number of new peers accepted from any single
+  source to at most a handful, to limit the effectiveness of malicious
+  peers wanting to trigger arbitrary outgoing connections or fill your
+  peer tables with junk data.
+* in the response to :func:`server.peers.subscribe` calls, consider
+  limiting the number of peers on similar IP subnets to protect
+  against sybil attacks, and in the case of onion servers the total
+  returned.
+* you should not advertise a peer's IP address if it also advertises a
+  hostname (avoiding duplicates).

docs/protocol.rst

@@ -0,0 +1,7 @@
+Electrum Protocol
+=================
+
+See rendered `protocol docs here <https://electrum-protocol.readthedocs.io/>`_,
+
+and `git repository here <https://github.com/spesmilo/electrum-protocol>`_.
+

docs/release_process.md

@@ -0,0 +1,33 @@
+# Release process
+
+### 1. Add changelog
+
+### 2. Update version number
+
+- `docs/conf.py`
+- `electrumx/__init__.py`
+
+### 3. Tag
+
+```
+$ git tag -s "$VERSION" -m "$VERSION"
+$ git push "$REMOTE_ORIGIN" tag "$VERSION"
+```
+
+### 4. Build sdist
+
+see [`contrib/sdist/`](contrib/sdist)
+
+```
+$ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./contrib/sdist/build.sh
+```
+
+### 5. Upload to PyPI
+
+```
+// $ python3 -m twine upload --repository testpypi dist/$DISTNAME
+$ python3 -m twine upload dist/$DISTNAME
+```
+
+This will prompt for a username and password. For the username, use `__token__`.
+For the password, use API token from https://pypi.org/manage/account/ , including the pypi- prefix.

docs/requirements-docs.txt

@@ -0,0 +1 @@
+Sphinx

docs/rpc-interface.rst

@@ -0,0 +1,280 @@
+RPC Interface
+=============
+
+You can query the status of a running server, and affect its behaviour
+by sending **JSON RPC** commands to the LocalRPC port it is listening
+on.  This is best done using the :file:`electrumx_rpc` script
+provided.
+
+The general form of invocation is::
+
+  electrumx_rpc [-p PORT] <command> [arg1 [arg2...]]
+
+The port to send the commands to can be specified on the command line,
+otherwise the environment variable :envvar:`RPC_PORT` is used, and if
+that is not set then **8000** is assumed.
+
+The following commands are available:
+
+add_peer
+--------
+
+Add a peer to the peers list.  ElectrumX will schedule an immediate
+connection attempt.  This command takes a single argument: the peer's
+"real name" as it used to advertise itself on IRC::
+
+  $ electrumx_rpc add_peer "ecdsa.net v1.0 s110 t"
+  "peer 'ecdsa.net v1.0 s110 t' added"
+
+daemon_url
+----------
+
+This command takes an optional argument that is interpreted
+identically to the :envvar:`DAEMON_URL` environment variable.  If
+omitted, the default argument value is the process's existing
+:envvar:`DAEMON_URL` environment variable.
+
+This command replaces the daemon's URL at run-time, and also
+forcefully rotates to the first URL in the list.
+
+For example, in case ElectrumX has previously failed over to a
+secondary daemon and you want to revert to the primary having resolved
+the connectivity issue, invoking this command without an argument will
+have that effect.
+
+disconnect
+----------
+
+Disconnect the given session IDs or group names.
+
+Session IDs can be obtained in the logs or with the `sessions`_ RPC command.  Group
+names can be obtained with the `groups`_ RPC command.
+
+The special string :const:`all` disconnects all sessions.
+
+Example::
+
+  $ electrumx_rpc disconnect 209.59.102 34 2
+  [
+      "disconnecting session 34",
+      "disconnecting group 209.59.102"
+      "unknown: 2",
+  ]
+
+getinfo
+-------
+
+Return a summary of server state.  This command takes no arguments.
+A typical result is as follows (with annotated comments)::
+
+  $ electrumx_rpc getinfo
+  {
+      "coin": "Bitcoin",
+      "daemon": "127.0.0.1:9334/",
+      "daemon height": 572154,         # The daemon's height when last queried
+      "db height": 572154,             # The height to which the DB is flushed
+      "groups": 586,                   # The number of session groups
+      "history cache": "185,014 lookups 9,756 hits 1,000 entries",
+      "merkle cache": "280 lookups 54 hits 213 entries",
+      "peers": {                       # Peer information
+          "bad": 1,
+          "good": 51,
+          "never": 2,
+          "stale": 0,
+          "total": 54
+      },
+      "pid": 11804,                    # Process ID
+      "request counts": {              # Count of RPC requests by method name
+          "blockchain.block.header": 245,
+          "blockchain.block.headers": 70,
+          "blockchain.estimatefee": 12776,
+          "blockchain.headers.subscribe": 2825,
+          "blockchain.relayfee": 740,
+          "blockchain.scripthash.get_history": 196,
+          "blockchain.scripthash.subscribe": 184626,
+          "blockchain.transaction.broadcast": 19,
+          "blockchain.transaction.get": 213,
+          "blockchain.transaction.get_merkle": 289,
+          "getinfo": 3,
+          "groups": 1,
+          "mempool.get_fee_histogram": 3194,
+          "mempool.get_info": 121,
+          "server.add_peer": 9,
+          "server.banner": 740,
+          "server.donation_address": 754,
+          "server.features": 50,
+          "server.peers.subscribe": 792,
+          "server.ping": 6412,
+          "server.version": 2866
+      },
+      "request total": 216820,         # Total requests served
+      "sessions": {                    # Live session stats
+          "count": 670,
+          "count with subs": 45,
+          "errors": 0,
+          "logged": 0,
+          "paused": 0,
+          "pending requests": 79,      # Number of requests currently being processed
+          "subs": 36292                # Total subscriptions
+      },
+      "tx hashes cache": "289 lookups 38 hits 213 entries",
+      "txs sent": 19,                  # Transactions broadcast
+      "uptime": "01h 39m 04s",
+      "version": "ElectrumX 1.10.1"
+  }
+
+Each ill-formed request, or one that does not follow the Electrum
+protocol, increments the error count of the session that sent it.
+
+:ref:`logging <session logging>` of sessions can be enabled by RPC.
+
+For more information on peers see :ref:`here <peers>`.
+
+Clients that are slow to consume data sent to them are :dfn:`paused` until their socket
+buffer drains sufficiently, at which point processing of requests resumes.
+
+Apart from very short intervals, typically after a new block or when a client has just
+connected, the number of unprocessed requests should be low, say 250 or fewer.  If it is
+over 1,000 the server is overloaded.
+
+Sessions are put into groups, primarily as an anti-DoS measure. Each session goes into
+a group based on its IP subnet (e.g. /24 for IPv4). Each member of a group incurs a fraction
+of the costs of the other group members. This appears in the `sessions_` list under the
+column XCost.
+
+groups
+------
+
+Return a list of all current session groups.  Takes no arguments.
+
+The output is quite similar to the `sessions`_ command.
+
+log
+---
+
+Toggle logging of the given session IDs or group names.  All incoming requests for a
+logged session are written to the server log.  The arguments are case-insensitive.
+
+When a group is specified, logging is toggled for its current members only; there is no
+effect on future group members.
+
+Session IDs can be obtained in the logs or with the `sessions`_ RPC command.  Group
+names can be obtained with the `groups`_ RPC command.
+
+The special string :const:`all` turns on logging of all current and future sessions,
+:const:`none` turns off logging of all current and future sessions, and :const:`new`
+toggles logging of future sessions.
+
+Example::
+
+  $ electrumx_rpc log new 6 t0 z
+  [
+    "logging new sessions",
+    "logging session 6",
+    "logging session 3",
+    "logging session 57",
+    "logging session 12"
+    "unknown: z",
+  ]
+
+In the above command sessions 3, 12 and 57 were in group `t0` (in fact, session 6 was
+too).
+
+.. _peers:
+
+peers
+-----
+
+Return a list of peer Electrum servers serving the same coin network.
+This command takes no arguments.
+
+Peer data is obtained via a peer discovery protocol documented
+:ref:`here <Peer Discovery>`::
+
+  $ electrumx_rpc peers
+  Host                           Status   TCP   SSL Server             Min  Max  Pruning   Last Good    Last Try Tries               Source IP Address
+  bch.tedy.pw                    good   50001 50002 ElectrumX 1.2.1    0.9  1.2          07h 29m 23s 07h 30m 40s     0                 peer 185.215.224.26
+  shsmithgoggryfbx.onion         good   60001 60002 ElectrumX 1.2.1    0.9  1.2          07h 30m 34s 07h 30m 38s     0                 peer
+  bccarihace4jdcnt.onion         good   52001 52002 ElectrumX 1.2.1    0.9  1.2          07h 30m 34s 07h 30m 39s     0                 peer
+  [...]
+  electroncash.checksum0.com     good   50001 50002 ElectrumX 1.2.1    0.9  1.1          07h 30m 40s 07h 30m 41s     0                 peer 149.56.198.233
+
+.. _query:
+
+query
+-----
+
+Run a query of the UTXO and history databases against one or more
+addresses, hex scripts or ASCII names (for coins that have an index
+on names like Namecoin).  `--limit <N>` or `-l <N>` limits the output
+for each kind to that many entries.  History is printed in blockchain
+order; UTXOs in an arbitrary order.
+
+For example::
+
+  $ electrumx_rpc query --limit 5 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac
+  Script: 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac
+  History #1: height 123,723 tx_hash 3387418aaddb4927209c5032f515aa442a6587d6e54677f08a03b8fa7789e688
+  History #2: height 127,280 tx_hash 4574958d135e66a53abf9c61950aba340e9e140be50efeea9456aa9f92bf40b5
+  History #3: height 127,909 tx_hash 8b960c87f9f1a6e6910e214fcf5f9c69b60319ba58a39c61f299548412f5a1c6
+  History #4: height 127,943 tx_hash 8f6b63012753005236b1b76e4884e4dee7415e05ab96604d353001662cde6b53
+  History #5: height 127,943 tx_hash 60ff2dfdf67917040139903a0141f7525a7d152365b371b35fd1cf83f1d7f704
+  UTXO #1: tx_hash 9aa497bf000b20f5ec5dc512bb6c1b60b68fc584d38b292b434e839ea8807bf0 tx_pos 0 height 254,148 value 5,500
+  UTXO #2: tx_hash 1c998142a5a5aae6f8c1eab245351413fe8d4032a3f14345f9943a0d0bc90ec0 tx_pos 0 height 254,161 value 5,500
+  UTXO #3: tx_hash 53345491b4829140be53f30079c6e4556a18545343b122900ebbfa158f9ca97a tx_pos 0 height 254,163 value 5,500
+  UTXO #4: tx_hash c71ad947ac46af217da3cd5521113cbd03e36ddada2b4452afe6c15f944d2529 tx_pos 0 height 372,916 value 1,000
+  UTXO #5: tx_hash c944a6acac054275a5e294e746d9ce79f6dcae91f3b4f5a84561aee6404a55b3 tx_pos 0 height 254,148 value 5,500
+  Balance: 17.8983303 BCH
+
+reorg
+-----
+
+Force a block chain reorganisation, primarily for debugging purposes.
+This command takes an optional argument - the number of blocks to
+reorg - which defaults to 3.
+
+That number of blocks will be backed up - using undo information
+stored in ElectrumX's database - and then ElectrumX will move forwards
+on the daemon's main chain to its current height.
+
+.. _sessions:
+
+sessions
+--------
+
+Return a list of all current sessions.  Takes no arguments::
+
+  ID     Flags            Client Proto    Cost   XCost  Reqs   Txs    Subs    Recv Recv KB    Sent Sent KB      Time                  Peer
+  1      S6                1.1.1   1.4       0      16     0     0       0       3       0       3       0    05m42s 165.255.191.213:22349
+  2      S6       all_seeing_eye   1.4       0      16     0     0       0       2       0       2       0    05m40s   67.170.52.226:24995
+  4      S6                3.3.2   1.4       0      16     0     0      34      45       5      45       3    05m40s 185.220.100.252:40463
+  3      S6                1.1.2   1.4       0      16     0     0       0       3       0       3       0    05m40s    89.17.142.28:59241
+
+The columns show information by session: the session ID, flags (see below), how the client
+identifies itself - typically the Electrum client version, the protocol version
+negotiated, the session cost, the additional session cost accrued from its groups, the
+number of unprocessed requests, the number of transactions sent, the number of address
+subscriptions, the number of requests received and their total size, the number of
+messages sent and their size, how long the client has been connected, and the client's IP
+address (if anonymous logging is disabled).
+
+The flags are:
+
+ * ``S`` an SSL connection
+ * ``T`` a TCP connection
+ * ``R`` a local RPC connection
+ * ``L`` a logged session
+ * ``C`` a connection that is being closed
+ * the non-negative number is the connection "cost", with lower
+   numbers having higher priority.  RPC connections have cost ``0``,
+   normal connections have cost at least ``1``.
+
+stop
+----
+
+Flush all cached data to disk and shut down the server cleanly, as if
+sending the `KILL` signal.  Be patient - during initial sync flushing
+all cached data to disk can take several minutes.  This command takes
+no arguments.
+
+.. _session logging:

electrumx_rpc

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 ; mode: python -*-
+import os
+import sys
+
+
+if __name__ == '__main__':
+    src_dir = os.path.join(os.path.dirname(__file__), "src")
+    sys.path.insert(0, src_dir)
+    from electrumx.cli.electrumx_rpc import main
+    sys.exit(main())

electrumx_server

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 ; mode: python -*-
+import os
+import sys
+
+
+if __name__ == '__main__':
+    src_dir = os.path.join(os.path.dirname(__file__), "src")
+    sys.path.insert(0, src_dir)
+    from electrumx.cli.electrumx_server import main
+    sys.exit(main())

pyproject.toml

@@ -0,0 +1,72 @@
+[build-system]
+requires = ["setuptools >= 61.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "e-x"
+authors = [
+    { name = "The Electrum developers" },
+]
+description = "Server implementation for the Electrum protocol"
+keywords = ["asyncio", "bitcoin", "electrum", "electrum-server"]
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENCE"]
+requires-python = ">=3.10"
+dependencies = [
+    "aiorpcx[ws]>=0.25.0,<0.26",
+    "aiohttp>=3.3,<4",
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Framework :: AsyncIO",
+    "Natural Language :: English",
+    "Operating System :: Unix",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Database",
+    "Topic :: Internet",
+]
+dynamic = ['version']
+
+[project.urls]
+Homepage = "https://github.com/spesmilo/electrumx"
+Repository = "https://github.com/spesmilo/electrumx"
+
+[project.scripts]
+electrumx_server = "electrumx.cli.electrumx_server:main"
+electrumx_rpc = "electrumx.cli.electrumx_rpc:main"
+
+[tool.setuptools.dynamic]
+version = { attr = 'electrumx.__version__' }
+
+[project.optional-dependencies]
+# DB_ENGINE choices. The user MUST install at least one.
+leveldb = [
+    "plyvel",
+]
+rocksdb = [
+    "rocksdb-ng",  # Felix's fork. ref https://github.com/f321x/python-rocksdb
+]
+# misc
+dev = [
+    "objgraph",
+]
+orjson = [
+    "orjson>=3,<4",
+]
+uvloop = [
+    "uvloop>=0.14",
+]
+# For various altcoins
+blake256 = ["blake256>=0.1.1", ]
+crypto = ["pycryptodomex>=3.8.1", ]
+groestl = ["groestlcoin-hash>=1.0.1", ]
+tribushashm = ["tribushashm>=1.0.5", ]
+xevan-hash = ["xevan-hash", ]
+dash_hash = ["dash_hash>=1.4", ]
+zny-yespower-0-5 = ["zny-yespower-0-5", ]

src/electrumx/__init__.py

@@ -0,0 +1,11 @@
+PYTHON_MIN_VERSION = (3, 10)  # noqa: E402
+
+
+from electrumx.server.controller import Controller
+from electrumx.server.env import Env
+
+
+BRANDING = "spesmilo"
+__version__ = "2.0.0"
+version = f'ElectrumX {__version__}'
+version_short = __version__

src/electrumx/cli/electrumx_rpc.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Script to send RPC commands to a running ElectrumX server.'''
+
+
+import argparse
+import asyncio
+import json
+import sys
+from os import environ
+
+from aiorpcx import timeout_after, connect_rs
+import electrumx.lib.text as text
+
+
+simple_commands = {
+    'getinfo': 'Print a summary of server state',
+    'groups': 'Print current session groups',
+    'peers': 'Print information about peer servers for the same coin',
+    'sessions': 'Print information about client sessions',
+    'stop': 'Shut down the server cleanly',
+}
+
+session_commands = {
+    'disconnect': 'Disconnect sessions',
+    'log': 'Control logging of sessions',
+}
+
+other_commands = {
+    'add_peer': (
+        'add a peer to the peers list',
+        [], {
+            'type': str,
+            'dest': 'real_name',
+            'help': 'e.g. "a.domain.name s995 t"',
+        },
+    ),
+    'daemon_url': (
+        "replace the daemon's URL at run-time, and forecefully rotate "
+        " to the first URL in the list",
+        [], {
+            'type': str,
+            'nargs': '?',
+            'default': '',
+            'dest': 'daemon_url',
+            'help': 'see documentation of DAEMON_URL envvar',
+        },
+    ),
+    'inspect_session': (
+        'inspect a given currently connected client session',
+        [], {
+            'type': int,
+            'dest': 'session_id',
+            'help': 'e.g. 1234 (see output of "sessions" command)',
+        },
+    ),
+    'query': (
+        'query the UTXO and history databases',
+        ['-l', '--limit'], {
+            'type': int,
+            'default': 1000,
+            'help': 'UTXO and history output limit',
+        }, ['items'], {
+            'nargs': '+',
+            'type': str,
+            'help': 'hex scripts, or addresses, to query',
+        },
+    ),
+    'reorg': (
+        'simulate a chain reorganization',
+        [], {
+            'type': int,
+            'dest': 'count',
+            'default': 3,
+            'help': 'number of blocks to back up'
+        },
+    ),
+    'debug_memusage_list_all_objects': (
+        'Print a table of types of most common types in memory',
+        ['--limit'], {
+            'type': int,
+            'default': 50,
+            'help': 'max number of types to return',
+        },
+    ),
+    'debug_memusage_get_random_backref_chain': (
+        "Return a dotfile as text containing the backref chain "
+        "for a randomly selected object of type objtype",
+        [], {
+            'type': str,
+            'dest': 'objtype',
+            'help': 'e.g. "_asyncio.Task"',
+        },
+    ),
+}
+
+
+def main():
+    '''Send the RPC command to the server and print the result.'''
+    main_parser = argparse.ArgumentParser(
+        'electrumx_rpc',
+        description='Send electrumx an RPC command'
+    )
+    main_parser.add_argument('-p', '--port', metavar='port_num', type=int,
+                             help='RPC port number')
+    main_parser.add_argument('--timeout', type=int, default=30,
+                             help='timeout for command in seconds')
+
+    subparsers = main_parser.add_subparsers(help='sub-command help',
+                                            dest='command')
+
+    for command, help in simple_commands.items():
+        parser = subparsers.add_parser(command, help=help)
+
+    for command, help in session_commands.items():
+        parser = subparsers.add_parser(command, help=help)
+        parser.add_argument('session_ids', nargs='+', type=str,
+                            help='list of session ids')
+
+    for command, data in other_commands.items():
+        parser_help, *arguments = data
+        parser = subparsers.add_parser(command, help=parser_help)
+        for n in range(0, len(arguments), 2):
+            args, kwargs = arguments[n: n+2]
+            parser.add_argument(*args, **kwargs)
+
+    args = main_parser.parse_args()
+    args = vars(args)
+    port = args.pop('port')
+    if port is None:
+        port = int(environ.get('RPC_PORT', 8000))
+    method = args.pop('command')
+    timeout = args.pop('timeout')
+
+    # aiorpcX makes this so easy...
+    async def send_request():
+        try:
+            async with timeout_after(timeout):
+                async with connect_rs('localhost', port) as session:
+                    session.transport._framer.max_size = 0
+                    session.sent_request_timeout = timeout
+                    result = await session.send_request(method, args)
+                    if method in ('query', ):
+                        for line in result:
+                            print(line)
+                    elif method in (
+                            'debug_memusage_list_all_objects',
+                            'debug_memusage_get_random_backref_chain',
+                    ):
+                        for line in result.split('\n'):
+                            print(line)
+                    elif method in ('groups', 'peers', 'sessions'):
+                        lines_func = getattr(text, f'{method}_lines')
+                        for line in lines_func(result):
+                            print(line)
+                    else:
+                        print(json.dumps(result, indent=4, sort_keys=True))
+            return 0
+        except OSError:
+            print('cannot connect - is ElectrumX catching up, not running, or '
+                  f'is {port} the wrong RPC port?')
+            return 1
+        except Exception as e:
+            print(f'error making request: {e}')
+            return 1
+
+    code = asyncio.run(send_request())
+    sys.exit(code)
+
+
+if __name__ == '__main__':
+    main()

src/electrumx/cli/electrumx_server.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Script to kick off the server.'''
+
+import asyncio
+import logging
+import sys
+
+import electrumx
+from electrumx import Controller, Env, PYTHON_MIN_VERSION
+from electrumx.lib.util import CompactFormatter, make_logger
+
+
+def main():
+    '''Set up logging and run the server.'''
+    log_fmt = Env.default('LOG_FORMAT', '%(levelname)s:%(name)s:%(message)s')
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(CompactFormatter(log_fmt))
+    logger = make_logger('electrumx', handler=handler, level='INFO')
+
+    logger.info(
+        f'ElectrumX server starting. '
+        f'({electrumx.BRANDING}. version={electrumx.__version__})')
+    try:
+        if sys.version_info < PYTHON_MIN_VERSION:
+            mvs = '.'.join(str(part) for part in PYTHON_MIN_VERSION)
+            raise RuntimeError(f'ElectrumX requires Python {mvs} or greater')
+        env = Env()
+        logger.info(f'logging level: {env.log_level}')
+        logger.setLevel(env.log_level)
+        controller = Controller(env)
+        asyncio.run(controller.run())
+    except Exception:
+        logger.exception('ElectrumX server terminated abnormally')
+    else:
+        logger.info('ElectrumX server terminated normally')
+
+
+if __name__ == '__main__':
+    main()

src/electrumx/lib/enum.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2016, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''An enum-like type with reverse lookup.
+
+Source: Python Cookbook, http://code.activestate.com/recipes/67107/
+'''
+
+
+class EnumError(Exception):
+    pass
+
+
+class Enumeration:
+
+    def __init__(self, name, enumList):
+        self.__doc__ = name
+
+        lookup = {}
+        reverseLookup = {}
+        i = 0
+        uniqueNames = set()
+        uniqueValues = set()
+        for x in enumList:
+            if isinstance(x, tuple):
+                x, i = x
+            if not isinstance(x, str):
+                raise EnumError("enum name {} not a string".format(x))
+            if not isinstance(i, int):
+                raise EnumError("enum value {} not an integer".format(i))
+            if x in uniqueNames:
+                raise EnumError("enum name {} not unique".format(x))
+            if i in uniqueValues:
+                raise EnumError("enum value {} not unique".format(x))
+            uniqueNames.add(x)
+            uniqueValues.add(i)
+            lookup[x] = i
+            reverseLookup[i] = x
+            i = i + 1
+        self.lookup = lookup
+        self.reverseLookup = reverseLookup
+
+    def __getattr__(self, attr):
+        result = self.lookup.get(attr)
+        if result is None:
+            raise AttributeError('enumeration has no member {}'.format(attr))
+        return result
+
+    def whatis(self, value):
+        return self.reverseLookup[value]

src/electrumx/lib/env_base.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2017, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Class for server environment configuration and defaults.'''
+
+
+from os import environ
+
+from electrumx.lib.util import class_logger
+
+
+class EnvBase:
+    '''Wraps environment configuration.'''
+
+    class Error(Exception):
+        pass
+
+    def __init__(self):
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.allow_root = self.boolean('ALLOW_ROOT', False)
+        self.loop_policy = self.event_loop_policy()
+
+    @classmethod
+    def default(cls, envvar, default):
+        return environ.get(envvar, default)
+
+    @classmethod
+    def boolean(cls, envvar, default):
+        default = 'Yes' if default else ''
+        return bool(cls.default(envvar, default).strip())
+
+    @classmethod
+    def required(cls, envvar):
+        value = environ.get(envvar)
+        if value is None:
+            raise cls.Error(f'required envvar {envvar} not set')
+        return value
+
+    @classmethod
+    def integer(cls, envvar, default):
+        value = environ.get(envvar)
+        if value is None:
+            return default
+        try:
+            return int(value)
+        except Exception:
+            raise cls.Error(f'cannot convert envvar {envvar} value {value} to '
+                            f'an integer')
+
+    @classmethod
+    def custom(cls, envvar, default, parse):
+        value = environ.get(envvar)
+        if value is None:
+            return default
+        try:
+            return parse(value)
+        except Exception as e:
+            raise cls.Error(
+                f'cannot parse envvar {envvar} value {value}'
+            ) from e
+
+    @classmethod
+    def obsolete(cls, envvars):
+        bad = [envvar for envvar in envvars if environ.get(envvar)]
+        if bad:
+            raise cls.Error(f'remove obsolete environment variables {bad}')
+
+    def event_loop_policy(self):
+        policy = self.default('EVENT_LOOP_POLICY', None)
+        if policy is None:
+            return None
+        if policy == 'uvloop':
+            import uvloop
+            return uvloop.EventLoopPolicy()
+        raise self.Error(f'unknown event loop policy "{policy}"')

src/electrumx/lib/hash.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Cryptograph hash functions and related classes.'''
+
+
+import hashlib
+import hmac
+
+from electrumx.lib.util import bytes_to_int, int_to_bytes, hex_to_bytes
+
+_sha256 = hashlib.sha256
+_new_hash = hashlib.new
+_hmac_digest = hmac.digest
+HASHX_LEN = 11
+
+
+def sha256(x: bytes) -> bytes:
+    '''Simple wrapper of hashlib sha256.'''
+    return _sha256(x).digest()
+
+
+def double_sha256(x: bytes) -> bytes:
+    '''SHA-256 of SHA-256, as used extensively in bitcoin.'''
+    return sha256(sha256(x))
+
+
+def hash_to_hex_str(x: bytes) -> str:
+    '''Convert a big-endian binary hash to displayed hex string. (rev-to-hum)
+
+    Display form of a binary hash is reversed and converted to hex.
+    '''
+    return bytes(reversed(x)).hex()
+
+
+def hex_str_to_hash(x: str) -> bytes:
+    '''Convert a displayed hex string to a binary hash. (hum-to-rev)'''
+    return bytes(reversed(hex_to_bytes(x)))
+
+
+class Base58Error(Exception):
+    '''Exception used for Base58 errors.'''
+
+
+class Base58:
+    '''Class providing base 58 functionality.'''
+
+    chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+    assert len(chars) == 58
+    cmap = {c: n for n, c in enumerate(chars)}
+
+    @staticmethod
+    def char_value(c):
+        val = Base58.cmap.get(c)
+        if val is None:
+            raise Base58Error(f'invalid base 58 character "{c}"')
+        return val
+
+    @staticmethod
+    def decode(txt):
+        """Decodes txt into a big-endian bytearray."""
+        if not isinstance(txt, str):
+            raise TypeError('a string is required')
+
+        if not txt:
+            raise Base58Error('string cannot be empty')
+
+        value = 0
+        for c in txt:
+            value = value * 58 + Base58.char_value(c)
+
+        result = int_to_bytes(value)
+
+        # Prepend leading zero bytes if necessary
+        count = 0
+        for c in txt:
+            if c != '1':
+                break
+            count += 1
+        if count:
+            result = bytes(count) + result
+
+        return result
+
+    @staticmethod
+    def encode(be_bytes):
+        """Converts a big-endian bytearray into a base58 string."""
+        value = bytes_to_int(be_bytes)
+
+        txt = ''
+        while value:
+            value, mod = divmod(value, 58)
+            txt += Base58.chars[mod]
+
+        for byte in be_bytes:
+            if byte != 0:
+                break
+            txt += '1'
+
+        return txt[::-1]
+
+    @staticmethod
+    def decode_check(txt, *, hash_fn=double_sha256):
+        '''Decodes a Base58Check-encoded string to a payload.  The version
+        prefixes it.'''
+        be_bytes = Base58.decode(txt)
+        result, check = be_bytes[:-4], be_bytes[-4:]
+        if check != hash_fn(result)[:4]:
+            raise Base58Error(f'invalid base 58 checksum for {txt}')
+        return result
+
+    @staticmethod
+    def encode_check(payload, *, hash_fn=double_sha256):
+        """Encodes a payload bytearray (which includes the version byte(s))
+        into a Base58Check string."""
+        be_bytes = payload + hash_fn(payload)[:4]
+        return Base58.encode(be_bytes)

src/electrumx/lib/lrucache.py

@@ -0,0 +1,190 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2022 Thomas Kemmer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# -----
+#
+# This is a stripped down LRU-cache from the "cachetools" library.
+# https://github.com/tkem/cachetools/blob/d991ac71b4eb6394be5ec572b835434081393215/src/cachetools/__init__.py
+
+import collections
+import collections.abc
+from typing import TypeVar, Dict
+
+
+class _DefaultSize:
+
+    __slots__ = ()
+
+    def __getitem__(self, _):
+        return 1
+
+    def __setitem__(self, _, value):
+        assert value == 1
+
+    def pop(self, _):
+        return 1
+
+
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
+
+
+class Cache(collections.abc.MutableMapping[_KT, _VT]):
+    """Mutable mapping to serve as a simple cache or cache base class."""
+
+    __marker = object()
+
+    __size = _DefaultSize()
+
+    def __init__(self, maxsize: int, getsizeof=None):
+        if getsizeof:
+            self.getsizeof = getsizeof
+        if self.getsizeof is not Cache.getsizeof:
+            self.__size = dict()
+        self.__data = dict()  # type: Dict[_KT, _VT]
+        self.__currsize = 0
+        self.__maxsize = maxsize
+        # these can be used externally to keep track of cache hit statistics:
+        # (proper internal bookkeeping w/o double-counting seems either expensive or complicated)
+        self.num_lookups = 0
+        self.num_hits = 0
+
+    def __repr__(self):
+        return "%s(%s, maxsize=%r, currsize=%r)" % (
+            self.__class__.__name__,
+            repr(self.__data),
+            self.__maxsize,
+            self.__currsize,
+        )
+
+    def __getitem__(self, key: _KT) -> _VT:
+        try:
+            return self.__data[key]
+        except KeyError:
+            return self.__missing__(key)
+
+    def __setitem__(self, key: _KT, value: _VT) -> None:
+        maxsize = self.__maxsize
+        size = self.getsizeof(value)
+        if size > maxsize:
+            raise ValueError("value too large")
+        if key not in self.__data or self.__size[key] < size:
+            while self.__currsize + size > maxsize:
+                self.popitem()
+        if key in self.__data:
+            diffsize = size - self.__size[key]
+        else:
+            diffsize = size
+        self.__data[key] = value
+        self.__size[key] = size
+        self.__currsize += diffsize
+
+    def __delitem__(self, key: _KT) -> None:
+        size = self.__size.pop(key)
+        del self.__data[key]
+        self.__currsize -= size
+
+    def __contains__(self, key: _KT) -> bool:
+        return key in self.__data
+
+    def __missing__(self, key: _KT):
+        raise KeyError(key)
+
+    def __iter__(self):
+        return iter(self.__data)
+
+    def __len__(self):
+        return len(self.__data)
+
+    def get(self, key: _KT, default: _VT = None) -> _VT | None:
+        if key in self:
+            return self[key]
+        else:
+            return default
+
+    def pop(self, key: _KT, default=__marker) -> _VT:
+        if key in self:
+            value = self[key]
+            del self[key]
+        elif default is self.__marker:
+            raise KeyError(key)
+        else:
+            value = default
+        return value
+
+    def setdefault(self, key: _KT, default: _VT = None) -> _VT | None:
+        if key in self:
+            value = self[key]
+        else:
+            self[key] = value = default
+        return value
+
+    @property
+    def maxsize(self) -> int:
+        """The maximum size of the cache."""
+        return self.__maxsize
+
+    @property
+    def currsize(self) -> int:
+        """The current size of the cache."""
+        return self.__currsize
+
+    @staticmethod
+    def getsizeof(value) -> int:
+        """Return the size of a cache element's value."""
+        return 1
+
+
+class LRUCache(Cache[_KT, _VT]):
+    """Least Recently Used (LRU) cache implementation."""
+
+    def __init__(self, maxsize: int, getsizeof=None):
+        Cache.__init__(self, maxsize, getsizeof)
+        self.__order = collections.OrderedDict()
+
+    def __getitem__(self, key: _KT, cache_getitem=Cache.__getitem__) -> _VT | None:
+        value = cache_getitem(self, key)
+        if key in self:  # __missing__ may not store item
+            self.__update(key)
+        return value
+
+    def __setitem__(self, key: _KT, value, cache_setitem=Cache.__setitem__) -> None:
+        cache_setitem(self, key, value)
+        self.__update(key)
+
+    def __delitem__(self, key: _KT, cache_delitem=Cache.__delitem__) -> None:
+        cache_delitem(self, key)
+        del self.__order[key]
+
+    def popitem(self) -> tuple[_KT, _VT]:
+        """Remove and return the `(key, value)` pair least recently used."""
+        try:
+            key = next(iter(self.__order))
+        except StopIteration:
+            raise KeyError("%s is empty" % type(self).__name__) from None
+        else:
+            return (key, self.pop(key))
+
+    def __update(self, key: _KT) -> None:
+        try:
+            self.__order.move_to_end(key)
+        except KeyError:
+            self.__order[key] = None

src/electrumx/lib/merkle.py

@@ -0,0 +1,266 @@
+# Copyright (c) 2018, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+'''Merkle trees, branches, proofs and roots.'''
+
+from math import ceil, log
+from typing import Sequence, Optional, Callable, Tuple, Iterable, List, Awaitable
+
+from aiorpcx import Event, run_in_thread
+
+from electrumx.lib.hash import double_sha256
+from electrumx.lib.util import class_logger
+
+
+class Merkle:
+    '''Perform merkle tree calculations on binary hashes using a given hash
+    function.
+
+    If the hash count is not even, the final hash is repeated when
+    calculating the next merkle layer up the tree.
+    '''
+
+    def __init__(self, hash_func: Callable[[bytes], bytes] = double_sha256):
+        self.hash_func = hash_func
+
+    def tree_depth(self, hash_count: int) -> int:
+        return self.branch_length(hash_count) + 1
+
+    def branch_length(self, hash_count: int) -> int:
+        '''Return the length of a merkle branch given the number of hashes.'''
+        if not isinstance(hash_count, int):
+            raise TypeError('hash_count must be an integer')
+        if hash_count < 1:
+            raise ValueError('hash_count must be at least 1')
+        return ceil(log(hash_count, 2))
+
+    def branch_and_root(
+            self, hashes: Iterable[bytes], index: int, length: Optional[int] = None,
+    ) -> Tuple[List[bytes], bytes]:
+        '''Return a (merkle branch, merkle_root) pair given hashes, and the
+        index of one of those hashes.
+        '''
+        hashes = list(hashes)
+        if not isinstance(index, int):
+            raise TypeError('index must be an integer')
+        # This also asserts hashes is not empty
+        if not 0 <= index < len(hashes):
+            raise ValueError('index out of range')
+        natural_length = self.branch_length(len(hashes))
+        if length is None:
+            length = natural_length
+        else:
+            if not isinstance(length, int):
+                raise TypeError('length must be an integer')
+            if length < natural_length:
+                raise ValueError('length out of range')
+
+        hash_func = self.hash_func
+        branch = []
+        for _ in range(length):
+            if len(hashes) & 1:
+                hashes.append(hashes[-1])
+            branch.append(hashes[index ^ 1])
+            index >>= 1
+            hashes = [hash_func(hashes[n] + hashes[n + 1])
+                      for n in range(0, len(hashes), 2)]
+
+        return branch, hashes[0]
+
+    def root(self, hashes: Sequence[bytes], length: Optional[int] = None) -> bytes:
+        '''Return the merkle root of a non-empty iterable of binary hashes.'''
+        _branch, root = self.branch_and_root(hashes, 0, length)
+        return root
+
+    def root_from_proof(self, hash: bytes, branch: Iterable[bytes], index: int) -> bytes:
+        '''Return the merkle root given a hash, a merkle branch to it, and
+        its index in the hashes array.
+
+        branch is an iterable sorted deepest to shallowest.  If the
+        returned root is the expected value then the merkle proof is
+        verified.
+
+        The caller should have confirmed the length of the branch with
+        branch_length().  Unfortunately this is not easily done for
+        bitcoin transactions as the number of transactions in a block
+        is unknown to an SPV client.
+        '''
+        hash_func = self.hash_func
+        for elt in branch:
+            if index & 1:
+                hash = hash_func(elt + hash)
+            else:
+                hash = hash_func(hash + elt)
+            index >>= 1
+        if index:
+            raise ValueError('index out of range for branch')
+        return hash
+
+    def level(self, hashes: Sequence[bytes], depth_higher: int) -> List[bytes]:
+        '''Return a level of the merkle tree of hashes the given depth
+        higher than the bottom row of the original tree.'''
+        size = 1 << depth_higher
+        root = self.root
+        return [root(hashes[n: n + size], depth_higher)
+                for n in range(0, len(hashes), size)]
+
+    def branch_and_root_from_level(
+            self, level: List[bytes], leaf_hashes: Sequence[bytes], index: int, depth_higher: int,
+    ) -> Tuple[List[bytes], bytes]:
+        '''Return a (merkle branch, merkle_root) pair when a merkle-tree has a
+        level cached.
+
+        To maximally reduce the amount of data hashed in computing a
+        merkle branch, cache a tree of depth N at level N // 2.
+
+        level is a list of hashes in the middle of the tree (returned
+        by level())
+
+        leaf_hashes are the leaves needed to calculate a partial branch
+        up to level.
+
+        depth_higher is how much higher level is than the leaves of the tree
+
+        index is the index in the full list of hashes of the hash whose
+        merkle branch we want.
+        '''
+        if not isinstance(level, list):
+            raise TypeError("level must be a list")
+        if not isinstance(leaf_hashes, list):
+            raise TypeError("leaf_hashes must be a list")
+        leaf_index = (index >> depth_higher) << depth_higher
+        leaf_branch, leaf_root = self.branch_and_root(
+            leaf_hashes, index - leaf_index, depth_higher)
+        index >>= depth_higher
+        level_branch, root = self.branch_and_root(level, index)
+        # Check last so that we know index is in-range
+        if leaf_root != level[index]:
+            raise ValueError('leaf hashes inconsistent with level')
+        return leaf_branch + level_branch, root
+
+
+class MerkleCache:
+    '''A cache to calculate merkle branches efficiently.'''
+
+    def __init__(
+            self,
+            merkle: 'Merkle',
+            source_func: Callable[[int, int], Awaitable[Sequence[bytes]]],
+    ):
+        '''Initialise a cache hashes taken from source_func:
+
+           async def source_func(index, count):
+              ...
+        '''
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.merkle = merkle
+        self.source_func = source_func
+        self.length = 0  # type: int
+        self.level = []  # type: List[bytes]
+        self.depth_higher = 0  # type: int
+        self.initialized = Event()
+
+    def _segment_length(self) -> int:
+        return 1 << self.depth_higher
+
+    def _leaf_start(self, index: int) -> int:
+        '''Given a level's depth higher and a hash index, return the leaf
+        index and leaf hash count needed to calculate a merkle branch.
+        '''
+        depth_higher = self.depth_higher
+        return (index >> depth_higher) << depth_higher
+
+    def _level(self, hashes: Sequence[bytes]) -> List[bytes]:
+        return self.merkle.level(hashes, self.depth_higher)
+
+    async def _extend_to(self, length: int) -> None:
+        '''Extend the length of the cache if necessary.'''
+        if length <= self.length:
+            return
+        # Start from the beginning of any final partial segment.
+        # Retain the value of depth_higher; in practice this is fine
+        start = self._leaf_start(self.length)
+        hashes = await self.source_func(start, length - start)
+        self.level[start >> self.depth_higher:] = self._level(hashes)
+        self.length = length
+
+    async def _level_for(self, length: int) -> List[bytes]:
+        '''Return a (level_length, final_hash) pair for a truncation
+        of the hashes to the given length.'''
+        if length == self.length:
+            return self.level
+        level = self.level[:length >> self.depth_higher]
+        leaf_start = self._leaf_start(length)
+        count = min(self._segment_length(), length - leaf_start)
+        hashes = await self.source_func(leaf_start, count)
+        level += self._level(hashes)
+        return level
+
+    async def initialize(self, length: int) -> None:
+        '''Call to initialize the cache to a source of given length.'''
+        self.length = length
+        self.depth_higher = self.merkle.tree_depth(length) // 2
+        hashes = await self.source_func(0, length)
+        self.level = await run_in_thread(self._level, hashes)
+        self.initialized.set()
+
+    def truncate(self, length: int) -> None:
+        '''Truncate the cache so it covers no more than length underlying
+        hashes.'''
+        if not isinstance(length, int):
+            raise TypeError('length must be an integer')
+        if length <= 0:
+            raise ValueError('length must be positive')
+        if length >= self.length:
+            return
+        length = self._leaf_start(length)
+        self.length = length
+        self.level[length >> self.depth_higher:] = []
+
+    async def branch_and_root(self, length: int, index: int) -> Tuple[List[bytes], bytes]:
+        '''Return a merkle branch and root.  Length is the number of
+        hashes used to calculate the merkle root, index is the position
+        of the hash to calculate the branch of.
+
+        index must be less than length, which must be at least 1.'''
+        if not isinstance(length, int):
+            raise TypeError('length must be an integer')
+        if not isinstance(index, int):
+            raise TypeError('index must be an integer')
+        if length <= 0:
+            raise ValueError('length must be positive')
+        if index >= length:
+            raise ValueError('index must be less than length')
+        await self.initialized.wait()
+        await self._extend_to(length)
+        leaf_start = self._leaf_start(index)
+        count = min(self._segment_length(), length - leaf_start)
+        leaf_hashes = await self.source_func(leaf_start, count)
+        if length < self._segment_length():
+            return self.merkle.branch_and_root(leaf_hashes, index)
+        level = await self._level_for(length)
+        return self.merkle.branch_and_root_from_level(
+            level, leaf_hashes, index, self.depth_higher)

src/electrumx/lib/peer.py

@@ -0,0 +1,337 @@
+# Copyright (c) 2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Representation of a peer server.'''
+
+from functools import cached_property
+from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network
+from socket import AF_INET, AF_INET6
+from typing import Any
+
+from aiorpcx import is_valid_hostname
+from electrumx.lib.util import protocol_tuple, version_string
+
+
+class Peer:
+
+    # Protocol version
+    ATTRS = ('host', 'features',
+             # metadata
+             'source', 'ip_addr',
+             'last_good', 'last_try', 'try_count')
+    FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max',
+                'ssl_port', 'tcp_port')
+    # This should be set by the application
+    DEFAULT_PORTS = {}
+
+    def __init__(self, host, features, source='unknown', ip_addr=None,
+                 last_good=0, last_try=0, try_count=0):
+        '''Create a peer given a host name (or IP address as a string),
+        a dictionary of features, and a record of the source.'''
+        assert isinstance(host, str)
+        assert isinstance(features, dict)
+        assert host in features.get('hosts', {})
+        self.host = host
+        self.features = features.copy()
+        # Canonicalize / clean-up
+        for feature in self.FEATURES:
+            self.features[feature] = getattr(self, feature)
+        # Metadata
+        self.source = source
+        self.ip_addr = ip_addr
+        # last_good represents the last connection that was
+        # successful *and* successfully verified, at which point
+        # try_count is set to 0.  Failure to connect or failure to
+        # verify increment the try_count.
+        self.last_good = last_good
+        self.last_try = last_try
+        self.try_count = try_count
+        # Transient, non-persisted metadata
+        self.bad = False
+        self.other_port_pairs = set()
+
+    @classmethod
+    def peers_from_features(cls, features: dict[str, Any] | Any, source):
+        peers = []
+        if isinstance(features, dict):
+            hosts = features.get('hosts')
+            if isinstance(hosts, dict):
+                peers = [Peer(host, features, source=source)
+                         for host in hosts if isinstance(host, str)]
+        return peers
+
+    @classmethod
+    def deserialize(cls, item):
+        '''Deserialize from a dictionary.'''
+        return cls(**item)
+
+    def matches(self, peers):
+        '''Return peers whose host matches our hostname or IP address.
+        Additionally include all peers whose IP address matches our
+        hostname if that is an IP address.
+        '''
+        candidates = (self.host.lower(), self.ip_addr)
+        return [peer for peer in peers
+                if peer.host.lower() in candidates
+                or peer.ip_addr == self.host]
+
+    def __str__(self):
+        return self.host
+
+    def update_features(self, features):
+        '''Update features in-place.'''
+        try:
+            tmp = Peer(self.host, features)
+        except AssertionError:
+            pass
+        else:
+            self.update_features_from_peer(tmp)
+
+    def update_features_from_peer(self, peer):
+        if peer != self:
+            self.features = peer.features
+            for feature in self.FEATURES:
+                setattr(self, feature, getattr(peer, feature))
+
+    def connection_tuples(self):
+        '''Return a list of (kind, port, family) tuples to try when making a
+        connection.
+        '''
+        # Use a list not a set - it's important to try the registered
+        # ports first.
+        pairs = [('SSL', self.ssl_port), ('TCP', self.tcp_port)]
+        while self.other_port_pairs:
+            pairs.append(self.other_port_pairs.pop())
+        if isinstance(self.ip_address, IPv4Address):
+            families = (AF_INET,)
+        elif isinstance(self.ip_address, IPv6Address):
+            families = (AF_INET6,)
+        else:
+            families = (AF_INET, AF_INET6)
+        return [(kind, port, family)
+                for kind, port in pairs if port
+                for family in families]
+
+    def mark_bad(self):
+        '''Mark as bad to avoid reconnects but also to remember for a
+        while.'''
+        self.bad = True
+
+    def check_ports(self, other):
+        '''Remember differing ports in case server operator changed them
+        or removed one.'''
+        if other.ssl_port != self.ssl_port:
+            self.other_port_pairs.add(('SSL', other.ssl_port))
+        if other.tcp_port != self.tcp_port:
+            self.other_port_pairs.add(('TCP', other.tcp_port))
+        return bool(self.other_port_pairs)
+
+    @cached_property
+    def is_tor(self):
+        return self.host.endswith('.onion')
+
+    @cached_property
+    def is_valid(self):
+        ip = self.ip_address
+        if ip:
+            return ((ip.is_global or ip.is_private)
+                    and not (ip.is_multicast or ip.is_unspecified))
+        return is_valid_hostname(self.host)
+
+    @cached_property
+    def is_public(self):
+        ip = self.ip_address
+        if ip:
+            return self.is_valid and not ip.is_private
+        else:
+            return self.is_valid and self.host != 'localhost'
+
+    @cached_property
+    def ip_address(self):
+        '''The host as a python ip_address object, or None.'''
+        try:
+            return ip_address(self.host)
+        except ValueError:
+            return None
+
+    def bucket_for_internal_purposes(self):
+        '''Used for keeping the internal peer list manageable in size.
+        Restrictions are loose.
+        '''
+        if self.is_tor:
+            return 'onion'
+        if not self.ip_addr:
+            return ''
+        ip_addr = ip_address(self.ip_addr)
+        if ip_addr.version == 4:
+            return str(ip_addr)
+        elif ip_addr.version == 6:
+            slash64 = IPv6Network(self.ip_addr).supernet(prefixlen_diff=128-64)
+            return str(slash64)
+        return ''
+
+    def bucket_for_external_interface(self):
+        '''Used when responding to RPC queries to return a distributed list
+        of peers. Restrictions are stricter than internal bucketing.
+        '''
+        if self.is_tor:
+            return 'onion'
+        if not self.ip_addr:
+            return ''
+        ip_addr = ip_address(self.ip_addr)
+        if ip_addr.version == 4:
+            slash16 = IPv4Network(self.ip_addr).supernet(prefixlen_diff=32-16)
+            return str(slash16)
+        elif ip_addr.version == 6:
+            slash56 = IPv6Network(self.ip_addr).supernet(prefixlen_diff=128-56)
+            return str(slash56)
+        return ''
+
+    def serialize(self):
+        '''Serialize to a dictionary.'''
+        return {attr: getattr(self, attr) for attr in self.ATTRS}
+
+    def _port(self, key):
+        hosts = self.features.get('hosts')
+        if isinstance(hosts, dict):
+            host = hosts.get(self.host)
+            port = self._integer(key, host)
+            if port and 0 < port < 65536:
+                return port
+        return None
+
+    def _integer(self, key, d=None):
+        d = d or self.features
+        result = d.get(key) if isinstance(d, dict) else None
+        if isinstance(result, str):
+            try:
+                result = int(result)
+            except ValueError:
+                pass
+        return result if isinstance(result, int) else None
+
+    def _string(self, key):
+        result = self.features.get(key)
+        return result if isinstance(result, str) else None
+
+    @cached_property
+    def genesis_hash(self):
+        '''Returns the network genesis block hash as a string if known, otherwise None.'''
+        return self._string('genesis_hash')
+
+    @cached_property
+    def ssl_port(self):
+        '''Returns None if no SSL port, otherwise the port as an integer.'''
+        return self._port('ssl_port')
+
+    @cached_property
+    def tcp_port(self):
+        '''Returns None if no TCP port, otherwise the port as an integer.'''
+        return self._port('tcp_port')
+
+    @cached_property
+    def server_version(self):
+        '''Returns the server version as a string if known, otherwise None.'''
+        return self._string('server_version')
+
+    @cached_property
+    def pruning(self):
+        '''Returns the pruning level as an integer.  None indicates no
+        pruning.'''
+        pruning = self._integer('pruning')
+        if pruning and pruning > 0:
+            return pruning
+        return None
+
+    def _protocol_version_string(self, key):
+        version_str = self.features.get(key)
+        ptuple = protocol_tuple(version_str)
+        return version_string(ptuple)
+
+    @cached_property
+    def protocol_min(self):
+        '''Minimum protocol version as a string, e.g., 1.0'''
+        return self._protocol_version_string('protocol_min')
+
+    @cached_property
+    def protocol_max(self):
+        '''Maximum protocol version as a string, e.g., 1.1'''
+        return self._protocol_version_string('protocol_max')
+
+    def to_tuple(self):
+        '''The tuple ((ip, host, details) expected in response
+        to a peers subscription.'''
+        details = self.real_name().split()[1:]
+        return (self.ip_addr or self.host, self.host, details)
+
+    def real_name(self):
+        '''Real name of this peer as used on IRC.'''
+        def port_text(letter, port):
+            if port == self.DEFAULT_PORTS.get(letter):
+                return letter
+            else:
+                return letter + str(port)
+
+        parts = [self.host, 'v' + self.protocol_max]
+        if self.pruning:
+            parts.append(f'p{self.pruning:d}')
+        for letter, port in (('s', self.ssl_port), ('t', self.tcp_port)):
+            if port:
+                parts.append(port_text(letter, port))
+        return ' '.join(parts)
+
+    @classmethod
+    def from_real_name(cls, real_name, source):
+        '''Real name is a real name as on IRC, such as
+
+            "erbium1.sytes.net v1.0 s t"
+
+        Returns an instance of this Peer class.
+        '''
+        host = 'nohost'
+        features = {}
+        ports = {}
+        for n, part in enumerate(real_name.split()):
+            if n == 0:
+                host = part
+                continue
+            if part[0] in ('s', 't'):
+                if len(part) == 1:
+                    port = cls.DEFAULT_PORTS[part[0]]
+                else:
+                    port = part[1:]
+                if part[0] == 's':
+                    ports['ssl_port'] = port
+                else:
+                    ports['tcp_port'] = port
+            elif part[0] == 'v':
+                features['protocol_max'] = features['protocol_min'] = part[1:]
+            elif part[0] == 'p':
+                features['pruning'] = part[1:]
+
+        features.update(ports)
+        features['hosts'] = {host: ports}
+
+        return cls(host, features, source)

src/electrumx/lib/script.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+'''Script-related classes and functions.'''
+from electrumx.lib.enum import Enumeration
+from electrumx.lib.util import unpack_le_uint16_from, unpack_le_uint32_from, \
+    pack_le_uint16, pack_le_uint32
+
+
+class ScriptError(Exception):
+    '''Exception used for script errors.'''
+
+
+OpCodes = Enumeration("Opcodes", [
+    ("OP_0", 0), ("OP_PUSHDATA1", 76),
+    "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE",
+    "OP_RESERVED",
+    "OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7", "OP_8",
+    "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
+    "OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF",
+    "OP_ELSE", "OP_ENDIF", "OP_VERIFY", "OP_RETURN",
+    "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP",
+    "OP_2OVER", "OP_2ROT", "OP_2SWAP", "OP_IFDUP", "OP_DEPTH", "OP_DROP",
+    "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL", "OP_ROT",
+    "OP_SWAP", "OP_TUCK",
+    "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE",
+    "OP_INVERT", "OP_AND", "OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY",
+    "OP_RESERVED1", "OP_RESERVED2",
+    "OP_1ADD", "OP_1SUB", "OP_2MUL", "OP_2DIV", "OP_NEGATE", "OP_ABS",
+    "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL", "OP_DIV", "OP_MOD",
+    "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR", "OP_NUMEQUAL",
+    "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN", "OP_GREATERTHAN",
+    "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
+    "OP_WITHIN",
+    "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160", "OP_HASH256",
+    "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
+    "OP_CHECKMULTISIGVERIFY",
+    "OP_NOP1",
+    "OP_CHECKLOCKTIMEVERIFY", "OP_CHECKSEQUENCEVERIFY"
+])
+
+
+# Paranoia to make it hard to create bad scripts
+assert OpCodes.OP_DUP == 0x76
+assert OpCodes.OP_HASH160 == 0xa9
+assert OpCodes.OP_EQUAL == 0x87
+assert OpCodes.OP_EQUALVERIFY == 0x88
+assert OpCodes.OP_CHECKSIG == 0xac
+assert OpCodes.OP_CHECKMULTISIG == 0xae
+
+
+def is_unspendable_legacy(script):
+    # OP_FALSE OP_RETURN or OP_RETURN
+    return script[:2] == b'\x00\x6a' or (script and script[0] == 0x6a)
+
+
+def is_unspendable_genesis(script):
+    # OP_FALSE OP_RETURN
+    return script[:2] == b'\x00\x6a'
+
+
+def _match_ops(ops, pattern):
+    if len(ops) != len(pattern):
+        return False
+    for op, pop in zip(ops, pattern):
+        if pop != op:
+            # -1 means 'data push', whose op is an (op, data) tuple
+            if pop == -1 and isinstance(op, tuple):
+                continue
+            return False
+
+    return True
+
+
+class ScriptPubKey:
+    '''A class for handling a tx output script that gives conditions
+    necessary for spending.
+    '''
+
+    TO_ADDRESS_OPS = (OpCodes.OP_DUP, OpCodes.OP_HASH160, -1,
+                      OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG)
+    TO_P2SH_OPS = (OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL)
+    TO_PUBKEY_OPS = (-1, OpCodes.OP_CHECKSIG)
+
+    @classmethod
+    def P2SH_script(cls, hash160):
+        return (bytes((OpCodes.OP_HASH160,))
+                + Script.push_data(hash160)
+                + bytes((OpCodes.OP_EQUAL,)))
+
+    @classmethod
+    def P2PKH_script(cls, hash160):
+        return (bytes((OpCodes.OP_DUP, OpCodes.OP_HASH160))
+                + Script.push_data(hash160)
+                + bytes((OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG)))
+
+
+class Script:
+
+    @classmethod
+    def get_ops(cls, script):
+        ops = []
+
+        # The unpacks or script[n] below throw on truncated scripts
+        try:
+            n = 0
+            while n < len(script):
+                op = script[n]
+                n += 1
+
+                if op <= OpCodes.OP_PUSHDATA4:
+                    # Raw bytes follow
+                    if op < OpCodes.OP_PUSHDATA1:
+                        dlen = op
+                    elif op == OpCodes.OP_PUSHDATA1:
+                        dlen = script[n]
+                        n += 1
+                    elif op == OpCodes.OP_PUSHDATA2:
+                        dlen, = unpack_le_uint16_from(script[n: n + 2])
+                        n += 2
+                    else:
+                        dlen, = unpack_le_uint32_from(script[n: n + 4])
+                        n += 4
+                    if n + dlen > len(script):
+                        raise IndexError
+                    op = (op, script[n:n + dlen])
+                    n += dlen
+
+                ops.append(op)
+        except Exception:
+            # Truncated script; e.g. tx_hash
+            # ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
+            raise ScriptError('truncated script')
+
+        return ops
+
+    @classmethod
+    def push_data(cls, data):
+        '''Returns the opcodes to push the data on the stack.'''
+        assert isinstance(data, (bytes, bytearray))
+
+        n = len(data)
+        if n < OpCodes.OP_PUSHDATA1:
+            return bytes((n,)) + data
+        if n < 256:
+            return bytes((OpCodes.OP_PUSHDATA1, n)) + data
+        if n < 65536:
+            return bytes((OpCodes.OP_PUSHDATA2,)) + pack_le_uint16(n) + data
+        return bytes((OpCodes.OP_PUSHDATA4,)) + pack_le_uint32(n) + data
+
+    @classmethod
+    def opcode_name(cls, opcode):
+        if OpCodes.OP_0 < opcode < OpCodes.OP_PUSHDATA1:
+            return f'OP_{opcode:d}'
+        try:
+            return OpCodes.whatis(opcode)
+        except KeyError:
+            return f'OP_UNKNOWN:{opcode:d}'
+
+    @classmethod
+    def dump(cls, script):
+        opcodes, datas = cls.get_ops(script)
+        for opcode, data in zip(opcodes, datas):
+            name = cls.opcode_name(opcode)
+            if data is None:
+                print(name)
+            else:
+                print(f'{name} {data.hex()} ({len(data):d} bytes)')

src/electrumx/lib/server_base.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2017, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Base class of servers'''
+
+import asyncio
+import os
+import platform
+import re
+import signal
+import sys
+import time
+from contextlib import suppress
+from functools import partial
+from typing import TYPE_CHECKING
+
+from aiorpcx import spawn
+
+from electrumx import PYTHON_MIN_VERSION
+from electrumx.lib.util import class_logger
+
+if TYPE_CHECKING:
+    from electrumx.server.env import Env
+
+
+class ServerBase:
+    '''Base class server implementation.
+
+    Derived classes are expected to:
+
+    - set SUPPRESS_MESSAGE_REGEX as appropriate
+    - implement the serve() coroutine, called from the run() method.
+      Upon return the event loop runs until the shutdown signal is received.
+    '''
+    SUPPRESS_MESSAGE_REGEX = re.compile('SSL handshake|Fatal read error on|'
+                                        'SSL error in data received|'
+                                        'socket.send() raised exception')
+    SUPPRESS_TASK_REGEX = re.compile('accept_connection2')
+
+    def __init__(self, env: 'Env'):
+        '''Save the environment, perform basic sanity checks, and set the
+        event loop policy.
+        '''
+        # First asyncio operation must be to set the event loop policy
+        # as this replaces the event loop
+        asyncio.set_event_loop_policy(env.loop_policy)
+
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        version_str = ' '.join(sys.version.splitlines())
+        self.logger.info(f'Python version: {version_str}')
+        self.env = env
+        self.start_time = 0
+
+        # Sanity checks
+        if sys.version_info < PYTHON_MIN_VERSION:
+            mvs = '.'.join(str(part) for part in PYTHON_MIN_VERSION)
+            raise RuntimeError(f'Python version >= {mvs} is required')
+
+        if platform.system() == 'Windows':
+            pass
+        elif os.geteuid() == 0 and not env.allow_root:
+            raise RuntimeError('RUNNING AS ROOT IS STRONGLY DISCOURAGED!\n'
+                               'You shoud create an unprivileged user account '
+                               'and use that.\n'
+                               'To continue as root anyway, restart with '
+                               'environment variable ALLOW_ROOT non-empty')
+
+    async def serve(self, shutdown_event: asyncio.Event):
+        '''Override to provide the main server functionality.
+        Run as a task that will be cancelled to request shutdown.
+
+        Setting the event also shuts down the server.
+        '''
+
+    def on_exception(self, loop, context):
+        '''Suppress spurious messages it appears we cannot control.'''
+        message = context.get('message')
+        if message and self.SUPPRESS_MESSAGE_REGEX.match(message):
+            return
+        if self.SUPPRESS_TASK_REGEX.match(repr(context.get('task'))):
+            return
+        loop.default_exception_handler(context)
+
+    async def run(self):
+        '''Run the server application:
+
+        - record start time
+        - install SIGINT and SIGTERM handlers to trigger shutdown_event
+        - set loop's exception handler to suppress unwanted messages
+        - run the event loop until serve() completes
+        '''
+        def on_signal(signame):
+            shutdown_event.set()
+            self.logger.warning(f'received {signame} signal, initiating shutdown')
+
+        async def serve():
+            try:
+                await self.serve(shutdown_event)
+            finally:
+                shutdown_event.set()
+
+        self.start_time = time.time()
+        loop = asyncio.get_event_loop()
+        shutdown_event = asyncio.Event()
+
+        if platform.system() != 'Windows':
+            # No signals on Windows
+            for signame in ('SIGINT', 'SIGTERM'):
+                loop.add_signal_handler(getattr(signal, signame),
+                                        partial(on_signal, signame))
+        loop.set_exception_handler(self.on_exception)
+
+        # Start serving and wait for shutdown, log receipt of the event
+        server_task = await spawn(serve, daemon=True)
+        try:
+            await shutdown_event.wait()
+        except KeyboardInterrupt:
+            self.logger.warning('received keyboard interrupt, initiating shutdown')
+
+        self.logger.info('shutting down')
+
+        server_task.cancel()
+        try:
+            with suppress(asyncio.CancelledError):
+                await server_task
+        finally:
+            self.logger.info('shutdown complete')

src/electrumx/lib/text.py

@@ -0,0 +1,86 @@
+import time
+
+import electrumx.lib.util as util
+
+
+def sessions_lines(data):
+    '''A generator returning lines for a list of sessions.
+
+    data is the return value of rpc_sessions().'''
+    fmt = ('{:<6} {:<5} {:>17} {:>5} '
+           '{:>7} {:>7} {:>5} {:>5} {:>7} '
+           '{:>7} {:>7} {:>7} {:>7} {:>9} {:>21}')
+    yield fmt.format('ID', 'Flags', 'Client', 'Proto',
+                     'Cost', 'XCost', 'Reqs', 'Txs', 'Subs',
+                     'Recv', 'Recv KB', 'Sent', 'Sent KB', 'Time', 'Peer')
+    for (id_, flags, peer, client, proto, cost, extra_cost, reqs, txs_sent, subs,
+         recv_count, recv_size, send_count, send_size, conn_time) in data:
+        yield fmt.format(id_, flags, client, proto,
+                         f'{int(cost):,d}',
+                         f'{int(extra_cost):,d}',
+                         f'{reqs:,d}',
+                         f'{txs_sent:,d}',
+                         f'{subs:,d}',
+                         f'{recv_count:,d}',
+                         f'{recv_size // 1024:,d}',
+                         f'{send_count:,d}',
+                         f'{send_size // 1024:,d}',
+                         util.formatted_time(conn_time, sep=''), peer)
+
+
+def groups_lines(data):
+    '''A generator returning lines for a list of groups.
+
+    data is the return value of rpc_groups().'''
+
+    fmt = ('{:<14} {:>9} {:>8} {:>8} {:>6} {:>6} {:>8}'
+           '{:>7} {:>9} {:>7} {:>9}')
+    yield fmt.format('Name', 'Sessions', 'Cost', 'Retained', 'Reqs', 'Txs', 'Subs',
+                     'Recv', 'Recv KB', 'Sent', 'Sent KB')
+    for (name, session_count, cost, retained_cost, reqs, txs_sent, subs,
+         recv_count, recv_size, send_count, send_size) in data:
+        yield fmt.format(name,
+                         f'{session_count:,d}',
+                         f'{int(cost):,d}',
+                         f'{int(retained_cost):,d}',
+                         f'{reqs:,d}',
+                         f'{txs_sent:,d}',
+                         f'{subs:,d}',
+                         f'{recv_count:,d}',
+                         f'{recv_size // 1024:,d}',
+                         f'{send_count:,d}',
+                         f'{send_size // 1024:,d}')
+
+
+def peers_lines(data):
+    '''A generator returning lines for a list of peers.
+
+    data is the return value of rpc_peers().'''
+    def time_fmt(t):
+        if not t:
+            return 'Never'
+        return util.formatted_time(now - t)
+
+    now = time.time()
+    fmt = ('{:<62} {:<6} {:>5} {:>5} {:<17} {:>4} '
+           '{:>4} {:>8} {:>11} {:>11} {:>5} {:>20} {:<15}')
+    yield fmt.format('Host', 'Status', 'TCP', 'SSL', 'Server', 'Min',
+                     'Max', 'Pruning', 'Last Good', 'Last Try',
+                     'Tries', 'Source', 'IP Address')
+    for item in data:
+        features = item['features']
+        hostname = item['host']
+        host = features['hosts'][hostname]
+        yield fmt.format(hostname[:62],
+                         item['status'],
+                         host.get('tcp_port') or '',
+                         host.get('ssl_port') or '',
+                         features['server_version'] or 'unknown',
+                         features['protocol_min'],
+                         features['protocol_max'],
+                         features['pruning'] or '',
+                         time_fmt(item['last_good']),
+                         time_fmt(item['last_try']),
+                         item['try_count'],
+                         item['source'][:20],
+                         item['ip_addr'] or '')

src/electrumx/lib/tx_axe.py

@@ -0,0 +1,489 @@
+# Copyright (c) 2016-2018, Neil Booth
+# Copyright (c) 2018, the ElectrumX authors
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Deserializer for AXE DIP2 special transaction types'''
+
+from dataclasses import dataclass
+from typing import Any
+
+from electrumx.lib.tx import Deserializer, Tx
+from electrumx.lib.util import (pack_le_uint16, pack_le_int32, pack_le_uint32,
+                                pack_le_int64, pack_varint, pack_varbytes,
+                                pack_be_uint16)
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002.md
+@dataclass(kw_only=True, slots=True)
+class AxeTx(Tx):
+    '''Class representing a Axe transaction'''
+    tx_type: int
+    extra_payload: Any
+
+    def serialize(self):
+        nLocktime = pack_le_uint32(self.locktime)
+        txins = (pack_varint(len(self.inputs)) +
+                 b''.join(tx_in.serialize() for tx_in in self.inputs))
+        txouts = (pack_varint(len(self.outputs)) +
+                  b''.join(tx_out.serialize() for tx_out in self.outputs))
+
+        if self.tx_type:
+            uVersion = pack_le_uint16(self.version)
+            uTxType = pack_le_uint16(self.tx_type)
+            vExtra = self._serialize_extra_payload()
+            return uVersion + uTxType + txins + txouts + nLocktime + vExtra
+        else:
+            nVersion = pack_le_int32(self.version)
+            return nVersion + txins + txouts + nLocktime
+
+    def _serialize_extra_payload(self):
+        extra = self.extra_payload
+        spec_tx_class = DeserializerAxe.SPEC_TX_HANDLERS.get(self.tx_type)
+        if not spec_tx_class:
+            assert isinstance(extra, (bytes, bytearray))
+            return pack_varbytes(extra)
+
+        if not isinstance(extra, spec_tx_class):
+            raise ValueError('Axe tx_type does not conform with extra'
+                             ' payload class: %s, %s' % (self.tx_type, extra))
+        return pack_varbytes(extra.serialize())
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002-special-transactions.md
+@dataclass(kw_only=True, slots=True)
+class AxeProRegTx:
+    '''Class representing DIP3 ProRegTx'''
+    version: int
+    type: int
+    mode: int
+    collateralOutpoint: 'TxOutPoint'
+    ipAddress: bytes
+    port: int
+    KeyIdOwner: bytes
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    operatorReward: int
+    scriptPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.ipAddress) == 16
+                and len(self.KeyIdOwner) == 20
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        return (
+            pack_le_uint16(self.version) +              # version
+            pack_le_uint16(self.type) +                 # type
+            pack_le_uint16(self.mode) +                 # mode
+            self.collateralOutpoint.serialize() +       # collateralOutpoint
+            self.ipAddress +                            # ipAddress
+            pack_be_uint16(self.port) +                 # port
+            self.KeyIdOwner +                           # KeyIdOwner
+            self.PubKeyOperator +                       # PubKeyOperator
+            self.KeyIdVoting +                          # KeyIdVoting
+            pack_le_uint16(self.operatorReward) +       # operatorReward
+            pack_varbytes(self.scriptPayout) +          # scriptPayout
+            self.inputsHash +                           # inputsHash
+            pack_varbytes(self.payloadSig)              # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProRegTx(
+            version=deser._read_le_uint16(),
+            type=deser._read_le_uint16(),
+            mode=deser._read_le_uint16(),
+            collateralOutpoint=deser._read_outpoint(),
+            ipAddress=deser._read_nbytes(16),
+            port=deser._read_be_uint16(),
+            KeyIdOwner=deser._read_nbytes(20),
+            PubKeyOperator=deser._read_nbytes(48),
+            KeyIdVoting=deser._read_nbytes(20),
+            operatorReward=deser._read_le_uint16(),
+            scriptPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_varbytes(),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeProUpServTx:
+    '''Class representing DIP3 ProUpServTx'''
+    version: int
+    proTxHash: bytes
+    ipAddress: bytes
+    port: int
+    scriptOperatorPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.ipAddress) == 16
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            self.ipAddress +                            # ipAddress
+            pack_be_uint16(self.port) +                 # port
+            pack_varbytes(self.scriptOperatorPayout) +  # scriptOperatorPayout
+            self.inputsHash +                           # inputsHash
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProUpServTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            ipAddress=deser._read_nbytes(16),
+            port=deser._read_be_uint16(),
+            scriptOperatorPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeProUpRegTx:
+    '''Class representing DIP3 ProUpRegTx'''
+    version: int
+    proTxHash: bytes
+    mode: int
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    scriptPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.mode) +                 # mode
+            self.PubKeyOperator +                       # PubKeyOperator
+            self.KeyIdVoting +                          # KeyIdVoting
+            pack_varbytes(self.scriptPayout) +          # scriptPayout
+            self.inputsHash +                           # inputsHash
+            pack_varbytes(self.payloadSig)              # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProUpRegTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            mode=deser._read_le_uint16(),
+            PubKeyOperator=deser._read_nbytes(48),
+            KeyIdVoting=deser._read_nbytes(20),
+            scriptPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_varbytes(),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeProUpRevTx:
+    '''Class representing DIP3 ProUpRevTx'''
+    version: int
+    proTxHash: bytes
+    reason: int
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.reason) +               # reason
+            self.inputsHash +                           # inputsHash
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProUpRevTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            reason=deser._read_le_uint16(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeCbTx:
+    '''Class representing DIP4 coinbase special tx'''
+    version: int
+    height: int
+    merkleRootMNList: bytes
+    merkleRootQuorums: bytes
+
+    def serialize(self):
+        assert len(self.merkleRootMNList) == 32
+        res = (
+            pack_le_uint16(self.version) +              # version
+            pack_le_uint32(self.height) +               # height
+            self.merkleRootMNList                       # merkleRootMNList
+        )
+        if self.version > 1:
+            assert len(self.merkleRootQuorums) == 32
+            res += self.merkleRootQuorums               # merkleRootQuorums
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()
+        height = deser._read_le_uint32()
+        merkleRootMNList = deser._read_nbytes(32)
+        merkleRootQuorums = b''
+        if version > 1:
+            merkleRootQuorums = deser._read_nbytes(32)
+        return AxeCbTx(
+            version=version,
+            height=height,
+            merkleRootMNList=merkleRootMNList,
+            merkleRootQuorums=merkleRootQuorums,
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxRegister:
+    '''Class representing DIP5 SubTxRegister'''
+    version: int
+    userName: bytes
+    pubKey: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.pubKey) == 48
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            pack_varbytes(self.userName) +              # userName
+            self.pubKey +                               # pubKey
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxRegister(
+            version=deser._read_le_uint16(),
+            userName=deser._read_varbytes(),
+            pubKey=deser._read_nbytes(48),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxTopup:
+    '''Class representing DIP5 SubTxTopup'''
+    version: int
+    regTxHash: bytes
+
+    def serialize(self):
+        assert len(self.regTxHash) == 32
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.regTxHash                              # regTxHash
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxTopup(
+            version=deser._read_le_uint16(),
+            regTxHash=deser._read_nbytes(32),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxResetKey:
+    '''Class representing DIP5 SubTxResetKey'''
+    version: int
+    regTxHash: bytes
+    hashPrevSubTx: bytes
+    creditFee: int
+    newPubKey: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.regTxHash) == 32
+                and len(self.hashPrevSubTx) == 32
+                and len(self.newPubKey) == 48
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.regTxHash +                            # regTxHash
+            self.hashPrevSubTx +                        # hashPrevSubTx
+            pack_le_int64(self.creditFee) +             # creditFee
+            self.newPubKey +                            # newPubKey
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxResetKey(
+            version=deser._read_le_uint16(),
+            regTxHash=deser._read_nbytes(32),
+            hashPrevSubTx=deser._read_nbytes(32),
+            creditFee=deser._read_le_int64(),
+            newPubKey=deser._read_nbytes(48),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxCloseAccount:
+    '''Class representing DIP5 SubTxCloseAccount'''
+    version: int
+    regTxHash: bytes
+    hashPrevSubTx: bytes
+    creditFee: int
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.regTxHash) == 32
+                and len(self.hashPrevSubTx) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.regTxHash +                            # regTxHash
+            self.hashPrevSubTx +                        # hashPrevSubTx
+            pack_le_int64(self.creditFee) +             # creditFee
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxCloseAccount(
+            version=deser._read_le_uint16(),
+            regTxHash=deser._read_nbytes(32),
+            hashPrevSubTx=deser._read_nbytes(32),
+            creditFee=deser._read_le_int64(),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+# https://dash-docs.github.io/en/developer-reference#outpoint
+@dataclass(kw_only=True, slots=True)
+class TxOutPoint:
+    '''Class representing tx output outpoint'''
+    hash: bytes
+    index: int
+
+    def serialize(self):
+        assert len(self.hash) == 32
+        return (
+            self.hash +                                 # hash
+            pack_le_uint32(self.index)                  # index
+        )
+
+    @classmethod
+    def read_outpoint(cls, deser):
+        return TxOutPoint(
+            hash=deser._read_nbytes(32),
+            index=deser._read_le_uint32(),
+        )
+
+
+class DeserializerAxe(Deserializer):
+    '''Deserializer for AXE DIP2 special tx types'''
+    # Supported Spec Tx types and corresponding classes mapping
+    PRO_REG_TX = 1
+    PRO_UP_SERV_TX = 2
+    PRO_UP_REG_TX = 3
+    PRO_UP_REV_TX = 4
+    CB_TX = 5
+    SUB_TX_REGISTER = 8
+    SUB_TX_TOPUP = 9
+    SUB_TX_RESET_KEY = 10
+    SUB_TX_CLOSE_ACCOUNT = 11
+
+    SPEC_TX_HANDLERS = {
+        PRO_REG_TX: AxeProRegTx,
+        PRO_UP_SERV_TX: AxeProUpServTx,
+        PRO_UP_REG_TX: AxeProUpRegTx,
+        PRO_UP_REV_TX: AxeProUpRevTx,
+        CB_TX: AxeCbTx,
+        SUB_TX_REGISTER: AxeSubTxRegister,
+        SUB_TX_TOPUP: AxeSubTxTopup,
+        SUB_TX_RESET_KEY: AxeSubTxResetKey,
+        SUB_TX_CLOSE_ACCOUNT: AxeSubTxCloseAccount,
+    }
+
+    def _read_outpoint(self):
+        return TxOutPoint.read_outpoint(self)
+
+    def read_tx(self):
+        start = self.cursor
+        header = self._read_le_uint32()
+        tx_type = header >> 16  # DIP2 tx type
+        if tx_type:
+            version = header & 0x0000ffff
+        else:
+            version = header
+
+        if tx_type and version < 3:
+            version = header
+            tx_type = 0
+
+        inputs = self._read_inputs()
+        outputs = self._read_outputs()
+        locktime = self._read_le_uint32()
+        if tx_type:
+            extra_payload_size = self._read_varint()
+            end = self.cursor + extra_payload_size
+            spec_tx_class = DeserializerAxe.SPEC_TX_HANDLERS.get(tx_type)
+            if spec_tx_class:
+                read_method = getattr(spec_tx_class, 'read_tx_extra', None)
+                extra_payload = read_method(self)
+                assert isinstance(extra_payload, spec_tx_class)
+            else:
+                extra_payload = self._read_nbytes(extra_payload_size)
+            assert self.cursor == end
+        else:
+            extra_payload = b''
+        txid = self.TX_HASH_FN(self.binary[start:self.cursor])
+        tx = AxeTx(
+            version=version,
+            inputs=inputs,
+            outputs=outputs,
+            locktime=locktime,
+            tx_type=tx_type,
+            extra_payload=extra_payload,
+            txid_rev=txid,
+            wtxid_rev=txid,
+        )
+        return tx

src/electrumx/lib/tx_dash.py

@@ -0,0 +1,442 @@
+# Copyright (c) 2016-2018, Neil Booth
+# Copyright (c) 2018, the ElectrumX authors
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Deserializer for Dash DIP2 special transaction types'''
+
+from dataclasses import dataclass
+from typing import Any
+
+from electrumx.lib.tx import Deserializer, Tx
+from electrumx.lib.util import (pack_le_uint16, pack_le_int32, pack_le_uint32,
+                                pack_le_int64, pack_varint, pack_varbytes,
+                                pack_be_uint16)
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002.md
+@dataclass(kw_only=True, slots=True)
+class DashTx(Tx):
+    '''Class representing a Dash transaction'''
+    tx_type: int
+    extra_payload: Any
+
+    def serialize(self):
+        nLocktime = pack_le_uint32(self.locktime)
+        txins = (pack_varint(len(self.inputs)) +
+                 b''.join(tx_in.serialize() for tx_in in self.inputs))
+        txouts = (pack_varint(len(self.outputs)) +
+                  b''.join(tx_out.serialize() for tx_out in self.outputs))
+
+        if self.tx_type:
+            uVersion = pack_le_uint16(self.version)
+            uTxType = pack_le_uint16(self.tx_type)
+            vExtra = self._serialize_extra_payload()
+            return uVersion + uTxType + txins + txouts + nLocktime + vExtra
+        else:
+            nVersion = pack_le_int32(self.version)
+            return nVersion + txins + txouts + nLocktime
+
+    def _serialize_extra_payload(self):
+        extra = self.extra_payload
+        spec_tx_class = DeserializerDash.SPEC_TX_HANDLERS.get(self.tx_type)
+        if not spec_tx_class:
+            assert isinstance(extra, (bytes, bytearray))
+            return pack_varbytes(extra)
+
+        if not isinstance(extra, spec_tx_class):
+            raise ValueError('Dash tx_type does not conform with extra'
+                             ' payload class: %s, %s' % (self.tx_type, extra))
+        return pack_varbytes(extra.serialize())
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002-special-transactions.md
+@dataclass(kw_only=True, slots=True)
+class DashProRegTx:
+    '''Class representing DIP3 ProRegTx'''
+    version: int
+    type: int
+    mode: int
+    collateralOutpoint: 'TxOutPoint'
+    ipAddress: bytes
+    port: int
+    KeyIdOwner: bytes
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    operatorReward: int
+    scriptPayout: bytes
+    inputsHash: bytes
+    platformNodeID: bytes
+    platformP2PPort: int
+    platformHTTPPort: int
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.ipAddress) == 16
+                and len(self.KeyIdOwner) == 20
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        res = (
+            pack_le_uint16(self.version) +                  # version
+            pack_le_uint16(self.type) +                     # type
+            pack_le_uint16(self.mode) +                     # mode
+            self.collateralOutpoint.serialize() +           # collateralOutpoint
+            self.ipAddress +                                # ipAddress
+            pack_be_uint16(self.port) +                     # port
+            self.KeyIdOwner +                               # KeyIdOwner
+            self.PubKeyOperator +                           # PubKeyOperator
+            self.KeyIdVoting +                              # KeyIdVoting
+            pack_le_uint16(self.operatorReward) +           # operatorReward
+            pack_varbytes(self.scriptPayout) +              # scriptPayout
+            self.inputsHash)                                # inputsHash
+        if self.version >= 2:
+            res += (self.platformNodeID +                   # platformNodeID
+                    pack_le_uint16(self.platformP2PPort) +  # platformP2PPort
+                    pack_le_uint16(self.platformHTTPPort))  # platformHTTPPort
+        res += pack_varbytes(self.payloadSig)               # payloadSig
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()               # version
+        ntype = deser._read_le_uint16()                 # type
+        mode = deser._read_le_uint16()                  # mode
+        collateralOutpoint = deser._read_outpoint()     # collateralOutpoint
+        ipAddress = deser._read_nbytes(16)              # ipAddress
+        port = deser._read_be_uint16()                  # port
+        KeyIdOwner = deser._read_nbytes(20)             # KeyIdOwner
+        PubKeyOperator = deser._read_nbytes(48)         # PubKeyOperator
+        KeyIdVoting = deser._read_nbytes(20)            # KeyIdVoting
+        operatorReward = deser._read_le_uint16()        # operatorReward
+        scriptPayout = deser._read_varbytes()           # scriptPayout
+        inputsHash = deser._read_nbytes(32)             # inputsHash
+        platformNodeID = b''
+        platformP2PPort = 0
+        platformHTTPPort = 0
+        if version >= 2 and ntype == 1:
+            platformNodeID = deser._read_nbytes(20)     # platformNodeID
+            platformP2PPort = deser._read_le_uint16()   # platformP2PPort
+            platformHTTPPort = deser._read_le_uint16()  # platformHTTPPort
+        payloadSig = deser._read_varbytes()             # payloadSig
+        return DashProRegTx(
+            version=version,
+            type=ntype,
+            mode=mode,
+            collateralOutpoint=collateralOutpoint,
+            ipAddress=ipAddress,
+            port=port,
+            KeyIdOwner=KeyIdOwner,
+            PubKeyOperator=PubKeyOperator,
+            KeyIdVoting=KeyIdVoting,
+            operatorReward=operatorReward,
+            scriptPayout=scriptPayout,
+            inputsHash=inputsHash,
+            platformNodeID=platformNodeID,
+            platformP2PPort=platformP2PPort,
+            platformHTTPPort=platformHTTPPort,
+            payloadSig=payloadSig,
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashProUpServTx:
+    '''Class representing DIP3 ProUpServTx'''
+    version: int
+    type: int
+    proTxHash: bytes
+    ipAddress: bytes
+    port: int
+    scriptOperatorPayout: bytes
+    inputsHash: bytes
+    platformNodeID: bytes
+    platformP2PPort: int
+    platformHTTPPort: int
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.ipAddress) == 16
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        res = pack_le_uint16(self.version)                  # version
+        if self.version >= 2:
+            res += pack_le_uint16(self.type)                # type
+        res += (
+            self.proTxHash +                                # proTxHash
+            self.ipAddress +                                # ipAddress
+            pack_be_uint16(self.port) +                     # port
+            pack_varbytes(self.scriptOperatorPayout) +      # scriptOperatorPayout
+            self.inputsHash)                                # inputsHash
+        if self.version >= 2:
+            res += (self.platformNodeID +                   # platformNodeID
+                    pack_le_uint16(self.platformP2PPort) +  # platformP2PPort
+                    pack_le_uint16(self.platformHTTPPort))  # platformHTTPPort
+        res += self.payloadSig                              # payloadSig
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()                   # version
+        ntype = 0
+        if version >= 2:
+            ntype = deser._read_le_uint16()                 # type
+        proTxHash = deser._read_nbytes(32)                  # proTxHash
+        ipAddress = deser._read_nbytes(16)                  # ipAddress
+        port = deser._read_be_uint16()                      # port
+        scriptOperatorPayout = deser._read_varbytes()       # scriptOperatorPayout
+        inputsHash = deser._read_nbytes(32)                 # inputsHash
+        platformNodeID = b''
+        platformP2PPort = 0
+        platformHTTPPort = 0
+        if version >= 2 and ntype == 1:
+            platformNodeID = deser._read_nbytes(20)         # platformNodeID
+            platformP2PPort = deser._read_le_uint16()       # platformP2PPort
+            platformHTTPPort = deser._read_le_uint16()      # platformHTTPPort
+        payloadSig = deser._read_nbytes(96)                 # payloadSig
+        return DashProUpServTx(
+            version=version,
+            type=ntype,
+            proTxHash=proTxHash,
+            ipAddress=ipAddress,
+            port=port,
+            scriptOperatorPayout=scriptOperatorPayout,
+            inputsHash=inputsHash,
+            platformNodeID=platformNodeID,
+            platformP2PPort=platformP2PPort,
+            platformHTTPPort=platformHTTPPort,
+            payloadSig=payloadSig,
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashProUpRegTx:
+    '''Class representing DIP3 ProUpRegTx'''
+    version: int
+    proTxHash: bytes
+    mode: int
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    scriptPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.mode) +                 # mode
+            self.PubKeyOperator +                       # PubKeyOperator
+            self.KeyIdVoting +                          # KeyIdVoting
+            pack_varbytes(self.scriptPayout) +          # scriptPayout
+            self.inputsHash +                           # inputsHash
+            pack_varbytes(self.payloadSig)              # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return DashProUpRegTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            mode=deser._read_le_uint16(),
+            PubKeyOperator=deser._read_nbytes(48),
+            KeyIdVoting=deser._read_nbytes(20),
+            scriptPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_varbytes(),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashProUpRevTx:
+    '''Class representing DIP3 ProUpRevTx'''
+    version: int
+    proTxHash: bytes
+    reason: int
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.reason) +               # reason
+            self.inputsHash +                           # inputsHash
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return DashProUpRevTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            reason=deser._read_le_uint16(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashCbTx:
+    '''Class representing DIP4 coinbase special tx'''
+    version: int
+    height: int
+    merkleRootMNList: bytes
+    merkleRootQuorums: bytes
+    bestCLHeightDiff: int
+    bestCLSignature: bytes
+    assetLockedAmount: int
+
+    def serialize(self):
+        assert len(self.merkleRootMNList) == 32
+        res = (
+            pack_le_uint16(self.version) +              # version
+            pack_le_uint32(self.height) +               # height
+            self.merkleRootMNList                       # merkleRootMNList
+        )
+        if self.version > 1:
+            assert len(self.merkleRootQuorums) == 32
+            res += self.merkleRootQuorums               # merkleRootQuorums
+        if self.version > 2:
+            res += pack_varint(self.bestCLHeightDiff)
+            res += self.bestCLSignature
+            res += pack_le_int64(self.assetLockedAmount)
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()
+        height = deser._read_le_uint32()
+        merkleRootMNList = deser._read_nbytes(32)
+        merkleRootQuorums = b''
+        bestCLHeightDiff = 0
+        bestCLSignature = b''
+        assetLockedAmount = 0
+        if version > 1:
+            merkleRootQuorums = deser._read_nbytes(32)
+        if version > 2:
+            bestCLHeightDiff = deser._read_varint()
+            bestCLSignature = deser._read_nbytes(96)
+            assetLockedAmount = deser._read_le_uint64()
+        return DashCbTx(
+            version=version,
+            height=height,
+            merkleRootMNList=merkleRootMNList,
+            merkleRootQuorums=merkleRootQuorums,
+            bestCLHeightDiff=bestCLHeightDiff,
+            bestCLSignature=bestCLSignature,
+            assetLockedAmount=assetLockedAmount,
+        )
+
+
+# https://dash-docs.github.io/en/developer-reference#outpoint
+@dataclass(kw_only=True, slots=True)
+class TxOutPoint:
+    '''Class representing tx output outpoint'''
+    hash: bytes
+    index: int
+
+    def serialize(self):
+        assert len(self.hash) == 32
+        return (
+            self.hash +                                 # hash
+            pack_le_uint32(self.index)                  # index
+        )
+
+    @classmethod
+    def read_outpoint(cls, deser):
+        return TxOutPoint(
+            hash=deser._read_nbytes(32),
+            index=deser._read_le_uint32(),
+        )
+
+
+class DeserializerDash(Deserializer):
+    '''Deserializer for Dash DIP2 special tx types'''
+    # Supported Spec Tx types and corresponding classes mapping
+    PRO_REG_TX = 1
+    PRO_UP_SERV_TX = 2
+    PRO_UP_REG_TX = 3
+    PRO_UP_REV_TX = 4
+    CB_TX = 5
+
+    SPEC_TX_HANDLERS = {
+        PRO_REG_TX: DashProRegTx,
+        PRO_UP_SERV_TX: DashProUpServTx,
+        PRO_UP_REG_TX: DashProUpRegTx,
+        PRO_UP_REV_TX: DashProUpRevTx,
+        CB_TX: DashCbTx,
+    }
+
+    def _read_outpoint(self):
+        return TxOutPoint.read_outpoint(self)
+
+    def read_tx(self):
+        start = self.cursor
+        header = self._read_le_uint32()
+        tx_type = header >> 16  # DIP2 tx type
+        if tx_type:
+            version = header & 0x0000ffff
+        else:
+            version = header
+
+        if tx_type and version < 3:
+            version = header
+            tx_type = 0
+
+        inputs = self._read_inputs()
+        outputs = self._read_outputs()
+        locktime = self._read_le_uint32()
+        if tx_type:
+            extra_payload_size = self._read_varint()
+            end = self.cursor + extra_payload_size
+            spec_tx_class = DeserializerDash.SPEC_TX_HANDLERS.get(tx_type)
+            if spec_tx_class:
+                read_method = getattr(spec_tx_class, 'read_tx_extra', None)
+                extra_payload = read_method(self)
+                assert isinstance(extra_payload, spec_tx_class)
+            else:
+                extra_payload = self._read_nbytes(extra_payload_size)
+            assert self.cursor == end
+        else:
+            extra_payload = b''
+        txid = self.TX_HASH_FN(self.binary[start:self.cursor])
+        tx = DashTx(
+            version=version,
+            inputs=inputs,
+            outputs=outputs,
+            locktime=locktime,
+            tx_type=tx_type,
+            extra_payload=extra_payload,
+            txid_rev=txid,
+            wtxid_rev=txid,
+        )
+        return tx

src/electrumx/lib/util.py

@@ -0,0 +1,468 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+'''Miscellaneous utility classes and functions.'''
+
+
+from array import array
+import asyncio
+import inspect
+from ipaddress import ip_address
+import logging
+import sys
+from collections.abc import Container, Mapping
+from struct import Struct
+import time
+from typing import Any, Optional
+
+import aiorpcx
+
+
+# Use orjson (faster 3rd party library) if available, fallback to stdlib
+try:
+    import orjson
+    json_deserialize = orjson.loads
+
+    def json_serialize(obj: Any) -> str:
+        return orjson.dumps(obj).decode()  # orjson.dumps() returns bytes
+except ImportError:
+    import json
+    json_deserialize = json.loads
+    json_serialize = json.dumps
+
+
+# Logging utilities
+
+
+class ConnectionLogger(logging.LoggerAdapter):
+    '''Prepends a connection identifier to a logging message.'''
+    def process(self, msg, kwargs):
+        conn_id = self.extra.get('conn_id', 'unknown')
+        return f'[{conn_id}] {msg}', kwargs
+
+
+class CompactFormatter(logging.Formatter):
+    '''Strips the module from the logger name to leave the class only.'''
+    def format(self, record):
+        record.name = record.name.rpartition('.')[-1]
+        return super().format(record)
+
+
+def make_logger(name, *, handler, level):
+    '''Return the root ElectrumX logger.'''
+    logger = logging.getLogger(name)
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    logger.propagate = False
+    return logger
+
+
+def class_logger(path, classname):
+    '''Return a hierarchical logger for a class.'''
+    return logging.getLogger(path).getChild(classname)
+
+
+def formatted_time(t, sep=' '):
+    '''Return a number of seconds as a string in days, hours, mins and
+    maybe secs.'''
+    t = int(t)
+    fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60))
+    parts = []
+    for fmt, n in fmts:
+        val = t // n
+        if parts or val:
+            parts.append(fmt.format(val))
+        t %= n
+    if len(parts) < 3:
+        parts.append(f'{t:02d}s')
+    return sep.join(parts)
+
+
+def deep_getsizeof(obj):
+    """Find the memory footprint of a Python object.
+
+    Based on code from code.tutsplus.com: http://goo.gl/fZ0DXK
+
+    This is a recursive function that drills down a Python object graph
+    like a dictionary holding nested dictionaries with lists of lists
+    and tuples and sets.
+
+    The sys.getsizeof function does a shallow size of only. It counts each
+    object inside a container as pointer only regardless of how big it
+    really is.
+    """
+
+    ids = set()
+
+    def size(o):
+        if id(o) in ids:
+            return 0
+
+        r = sys.getsizeof(o)
+        ids.add(id(o))
+
+        if isinstance(o, (str, bytes, bytearray, array)):
+            return r
+
+        if isinstance(o, Mapping):
+            return r + sum(size(k) + size(v) for k, v in o.items())
+
+        if isinstance(o, Container):
+            return r + sum(size(x) for x in o)
+
+        return r
+
+    return size(obj)
+
+
+def subclasses(base_class, strict=True):
+    '''Return a list of subclasses of base_class in its module.'''
+    def select(obj):
+        return (inspect.isclass(obj) and issubclass(obj, base_class) and
+                (not strict or obj != base_class))
+
+    pairs = inspect.getmembers(sys.modules[base_class.__module__], select)
+    return [pair[1] for pair in pairs]
+
+
+def chunks(items, size):
+    '''Break up items, an iterable, into chunks of length size.'''
+    for i in range(0, len(items), size):
+        yield items[i: i + size]
+
+
+def resolve_limit(limit):
+    if limit is None or limit < 0:
+        return -1
+    assert isinstance(limit, int)
+    return limit
+
+
+def bytes_to_int(be_bytes):
+    '''Interprets a big-endian sequence of bytes as an integer'''
+    return int.from_bytes(be_bytes, 'big')
+
+
+def int_to_bytes(value):
+    '''Converts an integer to a big-endian sequence of bytes'''
+    return value.to_bytes((value.bit_length() + 7) // 8, 'big')
+
+
+def increment_byte_string(bs):
+    '''Return the lexicographically next byte string of the same length.
+
+    Return None if there is none (when the input is all 0xff bytes).'''
+    try:
+        return (int.from_bytes(bs, 'big') + 1).to_bytes(len(bs), 'big')
+    except OverflowError:
+        return None
+
+
+class LogicalFile:
+    '''A logical binary file split across several separate files on disk.'''
+
+    def __init__(self, prefix, digits, file_size):
+        self.filename_fmt = f'{prefix}{{:0{digits:d}d}}'
+        self.file_size = file_size
+
+    def read(self, start, size=-1):
+        '''Read up to size bytes from the virtual file, starting at offset
+        start, and return them.
+
+        If size is -1 all bytes are read.'''
+        parts = []
+        while size != 0:
+            try:
+                with self.open_file(start, False) as f:
+                    part = f.read(size)
+                if not part:
+                    break
+            except FileNotFoundError:
+                break
+            parts.append(part)
+            start += len(part)
+            if size > 0:
+                size -= len(part)
+        return b''.join(parts)
+
+    def write(self, start, b):
+        '''Write the bytes-like object, b, to the underlying virtual file.'''
+        while b:
+            size = min(len(b), self.file_size - (start % self.file_size))
+            with self.open_file(start, True) as f:
+                f.write(b if size == len(b) else b[:size])
+            b = b[size:]
+            start += size
+
+    def open_file(self, start, create):
+        '''Open the virtual file and seek to start.  Return a file handle.
+        Raise FileNotFoundError if the file does not exist and create
+        is False.
+        '''
+        file_num, offset = divmod(start, self.file_size)
+        filename = self.filename_fmt.format(file_num)
+        f = open_file(filename, create)
+        f.seek(offset)
+        return f
+
+
+def open_file(filename, create=False):
+    '''Open the file name.  Return its handle.'''
+    try:
+        return open(filename, 'rb+')
+    except FileNotFoundError:
+        if create:
+            return open(filename, 'wb+')
+        raise
+
+
+def open_truncate(filename):
+    '''Open the file name.  Return its handle.'''
+    return open(filename, 'wb+')
+
+
+def address_string(address):
+    '''Return an address as a correctly formatted string.'''
+    host, port = address
+    try:
+        host = ip_address(host)
+    except ValueError:
+        pass
+    else:
+        if host.version == 6:
+            return f'[{host}]:{port:d}'
+    return f'{host}:{port:d}'
+
+
+def protocol_tuple(s):
+    '''Converts a protocol version number, such as "1.0" to a tuple (1, 0).
+
+    If the version number is bad, (0, ) indicating version 0 is returned.'''
+    try:
+        return tuple(int(part) for part in s.split('.'))
+    except (TypeError, ValueError, AttributeError):
+        return (0, )
+
+
+def version_string(ptuple):
+    '''Convert a version tuple such as (1, 2) to "1.2".
+    There is always at least one dot, so (1, ) becomes "1.0".'''
+    while len(ptuple) < 2:
+        ptuple += (0, )
+    return '.'.join(str(p) for p in ptuple)
+
+
+def protocol_version(client_req, min_tuple, max_tuple):
+    '''Given a client's protocol version string, return a pair of
+    protocol tuples:
+
+           (negotiated version, client min request)
+
+    If the request is unsupported, the negotiated protocol tuple is
+    None.
+    '''
+    if client_req is None:
+        client_min = client_max = min_tuple
+    else:
+        if isinstance(client_req, list) and len(client_req) == 2:
+            client_min, client_max = client_req
+        else:
+            client_min = client_max = client_req
+        client_min = protocol_tuple(client_min)
+        client_max = protocol_tuple(client_max)
+
+    result = min(client_max, max_tuple)
+    if result < max(client_min, min_tuple) or result == (0, ):
+        result = None
+
+    return result, client_min
+
+
+def is_hex_str(text: Any, *, allow_odd_len: bool = False) -> bool:
+    if not isinstance(text, str):
+        return False
+    if allow_odd_len and len(text) % 2 == 1:
+        return is_hex_str("0" + text, allow_odd_len=False)
+    try:
+        b = bytes.fromhex(text)
+    except Exception:
+        return False
+    # forbid whitespaces in text:
+    if len(text) != 2 * len(b):
+        return False
+    return True
+
+
+struct_le_i = Struct('<i')
+struct_le_q = Struct('<q')
+struct_le_H = Struct('<H')
+struct_le_I = Struct('<I')
+struct_le_Q = Struct('<Q')
+struct_be_H = Struct('>H')
+struct_be_I = Struct('>I')
+struct_be_Q = Struct('>Q')
+structB = Struct('B')
+
+unpack_le_int32_from = struct_le_i.unpack_from
+unpack_le_int64_from = struct_le_q.unpack_from
+unpack_le_uint16_from = struct_le_H.unpack_from
+unpack_le_uint32_from = struct_le_I.unpack_from
+unpack_le_uint64_from = struct_le_Q.unpack_from
+unpack_be_uint16_from = struct_be_H.unpack_from
+unpack_be_uint32_from = struct_be_I.unpack_from
+
+unpack_le_uint32 = struct_le_I.unpack
+unpack_le_uint64 = struct_le_Q.unpack
+unpack_be_uint32 = struct_be_I.unpack
+unpack_be_uint64 = struct_be_Q.unpack
+
+pack_le_int32 = struct_le_i.pack
+pack_le_int64 = struct_le_q.pack
+pack_le_uint16 = struct_le_H.pack
+pack_le_uint32 = struct_le_I.pack
+pack_le_uint64 = struct_le_Q.pack
+pack_be_uint16 = struct_be_H.pack
+pack_be_uint32 = struct_be_I.pack
+pack_be_uint64 = struct_be_Q.pack
+pack_byte = structB.pack
+
+hex_to_bytes = bytes.fromhex
+
+
+def pack_varint(n):
+    if n < 253:
+        return pack_byte(n)
+    if n < 65536:
+        return pack_byte(253) + pack_le_uint16(n)
+    if n < 4294967296:
+        return pack_byte(254) + pack_le_uint32(n)
+    return pack_byte(255) + pack_le_uint64(n)
+
+
+def pack_varbytes(data):
+    return pack_varint(len(data)) + data
+
+
+class OldTaskGroup(aiorpcx.TaskGroup):
+    """Automatically raises exceptions on join; as in aiorpcx prior to version 0.20"""
+    async def join(self):
+        if self._wait is all:
+            exc = False
+            try:
+                async for task in self:
+                    if not task.cancelled():
+                        task.result()
+            except BaseException:  # including asyncio.CancelledError
+                exc = True
+                raise
+            finally:
+                if exc:
+                    await self.cancel_remaining()
+                await super().join()
+        else:
+            await super().join()
+            if self.completed:
+                self.completed.result()
+
+
+# We monkey-patch aiorpcx TimeoutAfter (used by timeout_after and ignore_after API),
+# to fix a timing issue present in asyncio as a whole re timing out tasks.
+# To see the issue we are trying to fix, consider example:
+#     async def outer_task():
+#         async with timeout_after(0.1):
+#             await inner_task()
+# When the 0.1 sec timeout expires, inner_task will get cancelled by timeout_after
+# (=internal cancellation).
+# If around the same time (in terms of event loop iterations) another coroutine
+# cancels outer_task (=external cancellation), there will be a race.
+# Both cancellations work by propagating a CancelledError out to timeout_after, which then
+# needs to decide (in TimeoutAfter.__aexit__) whether it's due to an internal or external cancel.
+# AFAICT asyncio provides no reliable way of distinguishing between the two.
+# This patch tries to always give priority to external cancellations.
+# see https://github.com/kyuupichan/aiorpcX/issues/44
+# see https://github.com/aio-libs/async-timeout/issues/229
+# see https://bugs.python.org/issue42130 and https://bugs.python.org/issue45098
+def _aiorpcx_monkeypatched_set_new_deadline(task, deadline):
+    def timeout_task():
+        task._orig_cancel()
+        task._timed_out = None if getattr(task, "_externally_cancelled", False) else deadline
+
+    def mycancel(*args, **kwargs):
+        task._orig_cancel(*args, **kwargs)
+        task._externally_cancelled = True
+        task._timed_out = None
+    if not hasattr(task, "_orig_cancel"):
+        task._orig_cancel = task.cancel
+        task.cancel = mycancel
+    task._deadline_handle = task._loop.call_at(deadline, timeout_task)
+
+
+def _aiorpcx_monkeypatched_set_task_deadline(task, deadline):
+    ret = _aiorpcx_orig_set_task_deadline(task, deadline)
+    task._externally_cancelled = None
+    return ret
+
+
+def _aiorpcx_monkeypatched_unset_task_deadline(task):
+    if hasattr(task, "_orig_cancel"):
+        task.cancel = task._orig_cancel
+        del task._orig_cancel
+    return _aiorpcx_orig_unset_task_deadline(task)
+
+
+_aiorpcx_orig_set_task_deadline = aiorpcx.curio._set_task_deadline
+_aiorpcx_orig_unset_task_deadline = aiorpcx.curio._unset_task_deadline
+
+aiorpcx.curio._set_new_deadline = _aiorpcx_monkeypatched_set_new_deadline
+aiorpcx.curio._set_task_deadline = _aiorpcx_monkeypatched_set_task_deadline
+aiorpcx.curio._unset_task_deadline = _aiorpcx_monkeypatched_unset_task_deadline
+
+
+def get_running_loop() -> Optional[asyncio.AbstractEventLoop]:
+    """Returns the asyncio event loop that is *running in this thread*, if any."""
+    try:
+        return asyncio.get_running_loop()
+    except RuntimeError:
+        return None
+
+
+class LogTimeTaken:
+    """A simple context manager that logs how much time passed between enter and exit."""
+
+    def __init__(self, logger: logging.Logger, base_str: str, *, enabled: bool = True):
+        self.logger = logger
+        self.base_str = base_str
+        self.enabled = enabled
+        self.start_time = 0
+
+    def __enter__(self):
+        self.start_time = time.monotonic()
+        return None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        time_elapsed = time.monotonic() - self.start_time
+        if self.enabled:
+            self.logger.debug(f'{self.base_str} took {time_elapsed:.3f}s.')

src/electrumx/server/controller.py

@@ -0,0 +1,199 @@
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+from asyncio import Event
+from typing import Set, Dict, Tuple
+
+from aiorpcx import _version as aiorpcx_version
+
+import electrumx
+from electrumx.lib.server_base import ServerBase
+from electrumx.lib.util import version_string, OldTaskGroup
+from electrumx.lib.tx import TxOutpoint
+from electrumx.server.db import DB
+from electrumx.server.mempool import MemPool, MemPoolAPI
+from electrumx.server.session import SessionManager
+
+
+class Notifications:
+    # hashX notifications come from two sources: new blocks and
+    # mempool refreshes.
+    #
+    # A user with a pending transaction is notified after the block it
+    # gets in is processed.  Block processing can take an extended
+    # time, and the prefetcher might poll the daemon after the mempool
+    # code in any case.  In such cases the transaction will not be in
+    # the mempool after the mempool refresh.  We want to avoid
+    # notifying clients twice - for the mempool refresh and when the
+    # block is done.  This object handles that logic by deferring
+    # notifications appropriately.
+
+    def __init__(self):
+        self._touched_hashxs_mp = {}  # type: Dict[int, Set[bytes]]
+        self._touched_hashxs_bp = {}  # type: Dict[int, Set[bytes]]
+        self._touched_outpoints_mp = {}  # type: Dict[int, Set[TxOutpoint]]
+        self._touched_outpoints_bp = {}  # type: Dict[int, Set[TxOutpoint]]
+        self._highest_block = -1
+
+    async def _maybe_notify(self):
+        th_mp, th_bp = self._touched_hashxs_mp, self._touched_hashxs_bp
+        # figure out block height
+        common_heights = set(th_mp).intersection(th_bp)
+        if common_heights:
+            height = max(common_heights)
+        elif th_mp and max(th_mp) == self._highest_block:
+            height = self._highest_block
+        else:
+            # Either we are processing a block and waiting for it to
+            # come in, or we have not yet had a mempool update for the
+            # new block height
+            return
+        # hashXs
+        touched_hashxs = th_mp.pop(height)
+        for old in [h for h in th_mp if h <= height]:
+            del th_mp[old]
+        for old in [h for h in th_bp if h <= height]:
+            touched_hashxs.update(th_bp.pop(old))
+        # outpoints
+        to_mp, to_bp = self._touched_outpoints_mp, self._touched_outpoints_bp
+        touched_outpoints = to_mp.pop(height)
+        for old in [h for h in to_mp if h <= height]:
+            del to_mp[old]
+        for old in [h for h in to_bp if h <= height]:
+            touched_outpoints.update(to_bp.pop(old))
+
+        await self.notify(
+            height=height,
+            touched_hashxs=touched_hashxs,
+            touched_outpoints=touched_outpoints,
+        )
+
+    async def notify(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        pass
+
+    async def start(self, height: int, notify_func):
+        self._highest_block = height
+        self.notify = notify_func
+        await self.notify(
+            height=height,
+            touched_hashxs=set(),
+            touched_outpoints=set(),
+        )
+
+    async def on_mempool(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        self._touched_hashxs_mp[height] = touched_hashxs
+        self._touched_outpoints_mp[height] = touched_outpoints
+        await self._maybe_notify()
+
+    async def on_block(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        self._touched_hashxs_bp[height] = touched_hashxs
+        self._touched_outpoints_bp[height] = touched_outpoints
+        self._highest_block = height
+        await self._maybe_notify()
+
+
+class Controller(ServerBase):
+    '''Manages server initialisation and shutdown.
+
+    Servers are started once the mempool is synced after the block
+    processor first catches up with the daemon.
+    '''
+    async def serve(self, shutdown_event):
+        '''Start the RPC server and wait for the mempool to synchronize.  Then
+        start serving external clients.
+        '''
+        if not (0, 25, 0) <= aiorpcx_version < (0, 26):
+            raise RuntimeError(
+                f'aiorpcX version {aiorpcx_version!r} does not match required: 0.25.0<=ver<0.26')
+
+        env = self.env
+        min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
+        self.logger.info(f'software version: {electrumx.version}')
+        self.logger.info(f'aiorpcX version: {version_string(aiorpcx_version)}')
+        self.logger.info(f'supported protocol versions: {min_str}-{max_str}')
+        self.logger.info(f'event loop policy: {env.loop_policy}')
+        self.logger.info(f'reorg limit is {env.reorg_limit:,d} blocks')
+
+        notifications = Notifications()
+        Daemon = env.coin.DAEMON
+        BlockProcessor = env.coin.BLOCK_PROCESSOR
+
+        async with Daemon(env.coin, env.daemon_url) as daemon:
+            db = DB(env)
+            bp = BlockProcessor(env, db, daemon, notifications)
+
+            # Set notifications up to implement the MemPoolAPI
+            def get_db_height():
+                return db.db_height
+            notifications.height = daemon.height
+            notifications.db_height = get_db_height
+            notifications.db_height_changed = db.db_flushed_event.wait
+            notifications.cached_height = daemon.cached_height
+            notifications.daemon_height_changed = daemon.height_changed.wait
+            notifications.mempool_txids_hum = daemon.mempool_txids_hum
+            notifications.raw_transactions = daemon.getrawtransactions
+            notifications.lookup_utxos = db.lookup_utxos
+            MemPoolAPI.register(Notifications)
+            mempool = MemPool(
+                env.coin, notifications,
+                refresh_secs=env.daemon_poll_interval_mempool_msec/1000,
+            )
+
+            session_mgr = SessionManager(
+                env=env,
+                db=db,
+                block_processor=bp,
+                daemon=daemon,
+                mempool=mempool,
+                shutdown_event=shutdown_event,
+            )
+
+            # Test daemon authentication, and also ensure it has a cached
+            # height. Do this before entering the task group.
+            await daemon.height()
+
+            # Check if daemon is recent enough
+            # note: we only check the first daemon
+            #       (DAEMON_URL might be set to a list of daemons)
+            await daemon.check_daemon_version()
+            await daemon.check_daemon_indexes()
+
+            caught_up_event = Event()
+            mempool_event = Event()
+
+            async def wait_for_catchup():
+                await caught_up_event.wait()
+                await group.spawn(db.populate_header_merkle_cache())
+                await group.spawn(mempool.keep_synchronized(mempool_event))
+
+            try:
+                async with OldTaskGroup() as group:
+                    await group.spawn(session_mgr.serve(notifications, mempool_event))
+                    await group.spawn(bp.fetch_and_process_blocks(caught_up_event))
+                    await group.spawn(wait_for_catchup())
+            except Exception as e:
+                self.logger.exception("taskgroup died.")
+            finally:
+                self.logger.info("taskgroup stopped.")

src/electrumx/server/daemon.py

@@ -0,0 +1,638 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Class for handling asynchronous connections to a blockchain
+daemon.'''
+
+import asyncio
+import itertools
+import time
+from calendar import timegm
+from struct import pack
+from typing import TYPE_CHECKING, Type, Sequence, Any, Iterable, Optional
+
+import aiohttp
+from aiorpcx import JSONRPC
+
+from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash
+from electrumx.lib.tx import DeserializerDecred
+from electrumx.lib.util import (class_logger, hex_to_bytes, json_deserialize,
+                                json_serialize, pack_varint,
+                                unpack_le_uint16_from)
+
+if TYPE_CHECKING:
+    from electrumx.lib.coins import Coin
+
+
+class DaemonError(Exception):
+    '''Raised when the daemon returns an error in its results.'''
+
+
+class WarmingUpError(Exception):
+    '''Internal - when the daemon is warming up.'''
+
+
+class ServiceRefusedError(Exception):
+    '''Internal - when the daemon doesn't provide a JSON response, only an HTTP error, for
+    some reason.'''
+
+
+class DaemonParseError(Exception):
+    '''Internal - when the daemon returns an RPC_PARSE_ERROR.'''
+
+
+class Daemon:
+    '''Handles connections to a daemon at the given URL.'''
+
+    RPC_IN_WARMUP = -28
+    RPC_PARSE_ERROR = -32700
+
+    id_counter = itertools.count()
+
+    def __init__(
+            self,
+            coin: Type['Coin'],
+            url,
+            *,
+            max_workqueue=10,
+            init_retry=0.25,
+            max_retry=4.0,
+    ):
+        self.coin = coin
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.url_index = None
+        self.urls = []
+        self.set_url(url)
+        # Limit concurrent RPC calls to this number.
+        # See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16
+        self.workqueue_semaphore = asyncio.Semaphore(value=max_workqueue)
+        self.init_retry = init_retry
+        self.max_retry = max_retry
+        self._height = None
+        self.height_changed = asyncio.Event()
+        self.available_rpcs = {}
+        self.session = None
+
+        self._networkinfo_cache = (None, 0)
+        self._networkinfo_lock = asyncio.Lock()
+
+        self._mempoolinfo_cache = (None, 0)
+        self._mempoolinfo_lock = asyncio.Lock()
+
+    async def __aenter__(self):
+        self.session = aiohttp.ClientSession(connector=self.connector())
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.session.close()
+        self.session = None
+
+    def connector(self):
+        return None
+
+    async def check_daemon_version(self):
+        assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}"
+        if self.coin.MIN_REQUIRED_DAEMON_VERSION is None:
+            return
+
+        required_version = tuple(map(int, self.coin.MIN_REQUIRED_DAEMON_VERSION.split('.')))
+        network_info = await self.getnetworkinfo()
+        ni_version = network_info['version']
+        daemon_major = ni_version // 10_000
+        daemon_minor = (ni_version % 10_000) // 100
+        daemon_rev = ni_version % 100
+        daemon_version = (daemon_major, daemon_minor, daemon_rev)
+        if daemon_version < required_version:
+            raise RuntimeError(f"Bitcoin Core {daemon_version=} < {required_version=}.")
+
+    async def check_daemon_indexes(self):
+        assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}"
+        if not self.coin.REQUIRED_DAEMON_INDEXES:
+            return
+        index_info = await self.getindexinfo()
+        for req_index in self.coin.REQUIRED_DAEMON_INDEXES:
+            if req_index not in index_info:
+                raise RuntimeError(
+                    f"bitcoind missing required index: {req_index}. "
+                    f"You should set {req_index}=1 in your bitcoin.conf config file, and reindex.")
+            if not index_info[req_index]["synced"]:
+                # Should we raise? Not raising allows syncing a fresh bitcoind and e-x "in parallel".
+                self.logger.warning(f"bitcoind required index {req_index!r} is still syncing!")
+
+    def set_url(self, url: str) -> None:
+        '''Set the URLS to the given list, and switch to the first one.'''
+        urls = url.split(',')
+        urls = [self.coin.sanitize_url(url) for url in urls]
+        for n, url in enumerate(urls):
+            status = '' if n else ' (current)'
+            logged_url = self.logged_url(url)
+            self.logger.info(f'daemon #{n + 1} at {logged_url}{status}')
+        self.url_index = 0
+        self.urls = urls
+
+    def current_url(self) -> str:
+        '''Returns the current daemon URL.'''
+        return self.urls[self.url_index]
+
+    def logged_url(self, url=None):
+        '''The host and port part, for logging.'''
+        url = url or self.current_url()
+        return url[url.rindex('@') + 1:]
+
+    def failover(self):
+        '''Call to fail-over to the next daemon URL.
+
+        Returns False if there is only one, otherwise True.
+        '''
+        if len(self.urls) > 1:
+            self.url_index = (self.url_index + 1) % len(self.urls)
+            self.logger.info(f'failing over to {self.logged_url()}')
+            return True
+        return False
+
+    async def _send_data(self, data):
+        async with self.workqueue_semaphore:
+            async with self.session.post(self.current_url(), data=data) as resp:
+                kind = resp.headers.get('Content-Type', None)
+                if kind == 'application/json':
+                    return await resp.json(loads=json_deserialize)
+                text = await resp.text()
+                text = text.strip() or resp.reason
+                raise ServiceRefusedError(text)
+
+    async def _send(self, payload, processor):
+        '''Send a payload to be converted to JSON.
+
+        Handles temporary connection issues. Daemon response errors
+        are raise through DaemonError.
+        '''
+        def log_error(error):
+            nonlocal last_error_log, retry
+            now = time.monotonic()
+            if now - last_error_log > 60:
+                last_error_log = now
+                self.logger.error(f'{error}.  Retrying occasionally...')
+            if retry == self.max_retry and self.failover():
+                retry = 0
+
+        on_good_message = None
+        last_error_log = 0
+        data = json_serialize(payload)
+        retry = self.init_retry
+        while True:
+            try:
+                result = await self._send_data(data)
+                result = processor(result)
+                if on_good_message:
+                    self.logger.info(on_good_message)
+                return result
+            except asyncio.TimeoutError:
+                log_error('timeout error')
+            except aiohttp.ServerDisconnectedError:
+                log_error('disconnected')
+                on_good_message = 'connection restored'
+            except ConnectionResetError:
+                log_error('connection reset')
+                on_good_message = 'connection restored'
+            except aiohttp.ClientConnectionError:
+                log_error('connection problem - check your daemon is running')
+                on_good_message = 'connection restored'
+            except aiohttp.ClientError as e:
+                log_error(f'daemon error: {e}')
+                on_good_message = 'running normally'
+            except ServiceRefusedError as e:
+                log_error(f'daemon service refused: {e}')
+                on_good_message = 'running normally'
+            except WarmingUpError:
+                log_error('starting up checking blocks')
+                on_good_message = 'running normally'
+            except DaemonParseError as e:
+                log_error(f"received RPC_PARSE_ERROR ({e=!r}). request was: {data=!r}")
+                on_good_message = 'running normally'
+
+            await asyncio.sleep(retry)
+            retry = max(min(self.max_retry, retry * 2), self.init_retry)
+
+    async def _send_single(self, method, params=None):
+        '''Send a single request to the daemon.'''
+        def processor(result):
+            err = result['error']
+            if not err:
+                return result['result']
+            if err.get('code') == self.RPC_IN_WARMUP:
+                raise WarmingUpError
+            if err.get('code') == self.RPC_PARSE_ERROR:
+                raise DaemonParseError(err)
+            raise DaemonError(err)
+
+        payload = {'method': method, 'id': next(self.id_counter)}
+        if params:
+            payload['params'] = params
+        return await self._send(payload, processor)
+
+    async def _send_vector(self, method, params_iterable, replace_errs=False):
+        '''Send several requests of the same method.
+
+        The result will be an array of the same length as params_iterable.
+        If replace_errs is true, any item with an error is returned as None,
+        otherwise an exception is raised.'''
+        def processor(result):
+            errs = [item['error'] for item in result if item['error']]
+            if any(err.get('code') == self.RPC_IN_WARMUP for err in errs):
+                raise WarmingUpError
+            if any(err.get('code') == self.RPC_PARSE_ERROR for err in errs):
+                raise DaemonParseError
+            if not errs or replace_errs:
+                return [item['result'] for item in result]
+            raise DaemonError(errs)
+
+        payload = [{'method': method, 'params': p, 'id': next(self.id_counter)}
+                   for p in params_iterable]
+        if payload:
+            return await self._send(payload, processor)
+        return []
+
+    async def _is_rpc_available(self, method: str) -> bool:
+        '''Return whether given RPC method is available in the daemon.
+
+        Results are cached and the daemon will generally not be queried with
+        the same method more than once.'''
+        available = self.available_rpcs.get(method)
+        if available is None:
+            available = True
+            try:
+                await self._send_single(method)
+            except DaemonError as e:
+                err = e.args[0]
+                error_code = err.get("code")
+                available = error_code != JSONRPC.METHOD_NOT_FOUND
+            self.available_rpcs[method] = available
+        return available
+
+    async def block_hex_hashes(self, first: int, count: int) -> Sequence[str]:
+        '''Return the hex hashes (hum) of count block starting at height first.'''
+        params_iterable = ((h, ) for h in range(first, first + count))
+        return await self._send_vector('getblockhash', params_iterable)
+
+    async def deserialised_block(self, bhash_hum: str) -> dict:
+        '''Return the deserialised block with the given hex hash.'''
+        return await self._send_single('getblock', (bhash_hum, True))
+
+    async def raw_blocks(self, bhashes_hum: Sequence[str]) -> Sequence[bytes]:
+        '''Return the raw binary blocks with the given hex hashes.'''
+        params_iterable = ((h, False) for h in bhashes_hum)
+        blocks = await self._send_vector('getblock', params_iterable)
+        # Convert hex string to bytes
+        return [hex_to_bytes(block) for block in blocks]
+
+    async def mempool_txids_hum(self) -> Sequence[str]:
+        '''Update our record of the daemon's mempool hashes.'''
+        return await self._send_single('getrawmempool')
+
+    async def estimatefee(self, block_count, estimate_mode=None):
+        '''Return the fee estimate for the block count.  Units are whole
+        currency units per KB, e.g. 0.00000995, or -1 if no estimate
+        is available.
+        '''
+        if estimate_mode:
+            args = (block_count, estimate_mode)
+        else:
+            args = (block_count, )
+        if await self._is_rpc_available('estimatesmartfee'):
+            estimate = await self._send_single('estimatesmartfee', args)
+            return estimate.get('feerate', -1)
+        return await self._send_single('estimatefee', args)
+
+    async def getnetworkinfo(self):
+        '''Return the result of the 'getnetworkinfo' RPC call.'''
+        async with self._networkinfo_lock:
+            cache_val, cache_time = self._networkinfo_cache
+            if time.time() - cache_time < 60:  # seconds
+                return cache_val
+            val = await self._send_single('getnetworkinfo')
+            self._networkinfo_cache = (val, time.time())
+            return val
+
+    async def getmempoolinfo(self):
+        """Return the result of the 'getmempoolinfo' RPC call."""
+        async with self._mempoolinfo_lock:
+            cache_val, cache_time = self._mempoolinfo_cache
+            if time.time() - cache_time < 60:  # seconds
+                return cache_val
+            val = await self._send_single('getmempoolinfo')
+            self._mempoolinfo_cache = (val, time.time())
+            return val
+
+    async def getindexinfo(self):
+        """Return the result of the 'getindexinfo' RPC call."""
+        # note: not cached as it's not exposed to client sessions
+        return await self._send_single('getindexinfo')
+
+    async def relayfee(self):
+        """Same as getmempoolinfo['minrelaytxfee'].
+        The minimum fee required for a transaction to be relayed on by the daemon to the
+        bitcoin network. Doesn't guarantee mempool acceptance."""
+        network_info = await self.getnetworkinfo()
+        return network_info['relayfee']
+
+    async def mempool_info(self) -> dict[str, float]:
+        mempool_info = await self.getmempoolinfo()
+        return {
+            # Dynamic minimum fee rate in BTC/kvB for tx to be accepted.
+            # Is the maximum of minrelaytxfee and minimum mempool fee
+            'mempoolminfee': mempool_info['mempoolminfee'],
+            # Minimum relay fee for transactions in BTC/kvB, operator-configurable, static.
+            'minrelaytxfee': mempool_info['minrelaytxfee'],
+            # Minimum fee rate increment for mempool limiting or replacement in BTC/kvB,
+            # operator-configurable, static.
+            'incrementalrelayfee': mempool_info['incrementalrelayfee'],
+        }
+
+    async def getrawtransaction(self, txid_hum: str, verbose=False):
+        '''Return the serialized raw transaction with the given hash.'''
+        # Cast to int because some coin daemons are old and require it
+        return await self._send_single('getrawtransaction',
+                                       (txid_hum, int(verbose)))
+
+    async def getrawtransactions(self, txids_hum: Iterable[str], replace_errs=True) -> Sequence[bytes | None]:
+        '''Return the serialized raw transactions with the given hashes.
+
+        Replaces errors with None by default.'''
+        params_iterable = ((txid_hum, 0) for txid_hum in txids_hum)
+        txs = await self._send_vector('getrawtransaction', params_iterable,
+                                      replace_errs=replace_errs)
+        # Convert hex strings to bytes
+        return [hex_to_bytes(tx) if tx else None for tx in txs]
+
+    async def gettxspendingprevout(self, prev_txid_hum: str, txout_idx: int) -> dict[str, Any]:
+        """Query the daemon to find (if any) the spender of given outpoint."""
+        outpoints = [{"txid": prev_txid_hum, "vout": txout_idx}, ]
+        options = {"mempool_only": False}
+        tx_items = await self._send_single('gettxspendingprevout', (outpoints, options))
+        return tx_items[0]
+
+    async def broadcast_transaction(self, raw_tx: str) -> str:
+        '''Broadcast a transaction to the network.'''
+        return await self._send_single('sendrawtransaction', (raw_tx, ))
+
+    async def broadcast_package(self, raw_txs: Sequence[str]):
+        """Broadcast a package of transactions to the network using 'submitpackage'."""
+        return await self._send_single('submitpackage', (raw_txs, ))
+
+    async def testmempoolaccept(self, raw_txs: Sequence[str]):
+        """Query the daemon to test mempool acceptance of txs,
+        without adding them to the mempool or broadcasting them.
+        """
+        return await self._send_single('testmempoolaccept', (raw_txs, ))
+
+    async def height(self) -> int:
+        '''Query the daemon for its current height.'''
+        old_height = self._height
+        self._height = await self._send_single('getblockcount')
+        if old_height != self._height:
+            self.logger.debug(f"new height: {self._height}")
+            self.height_changed.set()
+            self.height_changed.clear()
+        return self._height
+
+    def cached_height(self) -> Optional[int]:
+        '''Return the cached daemon height.
+
+        If the daemon has not been queried yet this returns None.'''
+        return self._height
+
+
+class DashDaemon(Daemon):
+
+    async def masternode_broadcast(self, params):
+        '''Broadcast a transaction to the network.'''
+        return await self._send_single('masternodebroadcast', params)
+
+    async def masternode_list(self, params):
+        '''Return the masternode status.'''
+        return await self._send_single('masternodelist', params)
+
+    async def protx(self, params):
+        '''Set of commands to execute ProTx related actions.'''
+        return await self._send_single('protx', params)
+
+
+class FakeEstimateFeeDaemon(Daemon):
+    '''Daemon that simulates estimatefee and relayfee RPC calls. Coin that
+    wants to use this daemon must define ESTIMATE_FEE & RELAY_FEE'''
+
+    async def estimatefee(self, block_count):
+        '''Return the fee estimate for the given parameters.'''
+        return self.coin.ESTIMATE_FEE
+
+    async def relayfee(self):
+        """The minimum fee required for a transaction to be relayed on by the server to the
+        bitcoin network. Doesn't guarantee mempool acceptance."""
+        return self.coin.RELAY_FEE
+
+
+class LegacyRPCDaemon(Daemon):
+    '''Handles connections to a daemon at the given URL.
+
+    This class is useful for daemons that don't have the new 'getblock'
+    RPC call that returns the block in hex, the workaround is to manually
+    recreate the block bytes. The recreated block bytes may not be the exact
+    as in the underlying blockchain but it is good enough for our indexing
+    purposes.'''
+
+    async def raw_blocks(self, bhashes_hum):
+        '''Return the raw binary blocks with the given hex hashes.'''
+        params_iterable = ((h, ) for h in bhashes_hum)
+        block_info = await self._send_vector('getblock', params_iterable)
+
+        blocks = []
+        for i in block_info:
+            raw_block = await self.make_raw_block(i)
+            blocks.append(raw_block)
+
+        # Convert hex string to bytes
+        return blocks
+
+    async def make_raw_header(self, b):
+        pbh = b.get('previousblockhash')
+        if pbh is None:
+            pbh = '0' * 64
+        return b''.join([
+            pack('<L', b.get('version')),
+            hex_str_to_hash(pbh),
+            hex_str_to_hash(b.get('merkleroot')),
+            pack('<L', self.timestamp_safe(b['time'])),
+            pack('<L', int(b.get('bits'), 16)),
+            pack('<L', int(b.get('nonce')))
+        ])
+
+    async def make_raw_block(self, b):
+        '''Construct a raw block'''
+
+        header = await self.make_raw_header(b)
+
+        transactions = []
+        if b.get('height') > 0:
+            transactions = await self.getrawtransactions(b.get('tx'), False)
+
+        raw_block = header
+        num_txs = len(transactions)
+        if num_txs > 0:
+            raw_block += pack_varint(num_txs)
+            raw_block += b''.join(transactions)
+        else:
+            raw_block += b'\x00'
+
+        return raw_block
+
+    def timestamp_safe(self, t):
+        if isinstance(t, int):
+            return t
+        return timegm(time.strptime(t, "%Y-%m-%d %H:%M:%S %Z"))
+
+
+class FakeEstimateLegacyRPCDaemon(LegacyRPCDaemon, FakeEstimateFeeDaemon):
+    pass
+
+
+class DecredDaemon(Daemon):
+    async def raw_blocks(self, bhashes_hum):
+        '''Return the raw binary blocks with the given hex hashes.'''
+
+        params_iterable = ((h, False) for h in bhashes_hum)
+        blocks = await self._send_vector('getblock', params_iterable)
+
+        raw_blocks = []
+        valid_tx_tree = {}
+        for block in blocks:
+            # Convert to bytes from hex
+            raw_block = hex_to_bytes(block)
+            raw_blocks.append(raw_block)
+            # Check if previous block is valid
+            prev = self.prev_hex_hash(raw_block)
+            votebits = unpack_le_uint16_from(raw_block[100:102])[0]
+            valid_tx_tree[prev] = self.is_valid_tx_tree(votebits)
+
+        processed_raw_blocks = []
+        for hash, raw_block in zip(bhashes_hum, raw_blocks):
+            if hash in valid_tx_tree:
+                is_valid = valid_tx_tree[hash]
+            else:
+                # Do something complicated to figure out if this block is valid
+                header = await self._send_single('getblockheader', (hash, ))
+                if 'nextblockhash' not in header:
+                    raise DaemonError(f'Could not find next block for {hash}')
+                next_hash = header['nextblockhash']
+                next_header = await self._send_single('getblockheader',
+                                                      (next_hash, ))
+                is_valid = self.is_valid_tx_tree(next_header['votebits'])
+
+            if is_valid:
+                processed_raw_blocks.append(raw_block)
+            else:
+                # If this block is invalid remove the normal transactions
+                self.logger.info(f'block {hash} is invalidated')
+                processed_raw_blocks.append(self.strip_tx_tree(raw_block))
+
+        return processed_raw_blocks
+
+    @staticmethod
+    def prev_hex_hash(raw_block):
+        return hash_to_hex_str(raw_block[4:36])
+
+    @staticmethod
+    def is_valid_tx_tree(votebits):
+        # Check if previous block was invalidated.
+        return bool(votebits & (1 << 0) != 0)
+
+    def strip_tx_tree(self, raw_block):
+        c = self.coin
+        assert issubclass(c.DESERIALIZER, DeserializerDecred)
+        d = c.DESERIALIZER(raw_block, start=c.BASIC_HEADER_SIZE)
+        d.read_tx_tree()  # Skip normal transactions
+        # Create a fake block without any normal transactions
+        return raw_block[:c.BASIC_HEADER_SIZE] + b'\x00' + raw_block[d.cursor:]
+
+    async def height(self):
+        height = await super().height()
+        if height > 0:
+            # Lie about the daemon height as the current tip can be invalidated
+            height -= 1
+            self._height = height
+        return height
+
+    async def mempool_txids_hum(self):
+        mempool = await super().mempool_txids_hum()
+        # Add current tip transactions to the 'fake' mempool.
+        real_height = await self._send_single('getblockcount')
+        tip_hash = await self._send_single('getblockhash', (real_height,))
+        tip = await self.deserialised_block(tip_hash)
+        # Add normal transactions except coinbase
+        mempool += tip['tx'][1:]
+        # Add stake transactions if applicable
+        mempool += tip.get('stx', [])
+        return mempool
+
+    def connector(self):
+        # FIXME allow self signed certificates
+        return aiohttp.TCPConnector(verify_ssl=False)
+
+
+class PreLegacyRPCDaemon(LegacyRPCDaemon):
+    '''Handles connections to a daemon at the given URL.
+
+    This class is useful for daemons that don't have the new 'getblock'
+    RPC call that returns the block in hex, and need the False parameter
+    for the getblock'''
+
+    async def deserialised_block(self, hex_hash):
+        '''Return the deserialised block with the given hex hash.'''
+        return await self._send_single('getblock', (hex_hash, False))
+
+
+class SmartCashDaemon(Daemon):
+
+    async def masternode_broadcast(self, params):
+        '''Broadcast a smartnode to the network.'''
+        return await self._send_single('smartnodebroadcast', params)
+
+    async def masternode_list(self, params):
+        '''Return the smartnode status.'''
+        return await self._send_single('smartnodelist', params)
+
+    async def smartrewards(self, params):
+        '''Return smartrewards data.'''
+        return await self._send_single('smartrewards', params)
+
+
+class ZcoinMtpDaemon(Daemon):
+
+    def strip_mtp_data(self, raw_block):
+        if self.coin.is_mtp(raw_block):
+            return \
+                raw_block[:self.coin.MTP_HEADER_DATA_START*2] + \
+                raw_block[self.coin.MTP_HEADER_DATA_END*2:]
+        return raw_block
+
+    async def raw_blocks(self, bhashes_hum):
+        '''Return the raw binary blocks with the given hex hashes.'''
+        params_iterable = ((h, False) for h in bhashes_hum)
+        blocks = await self._send_vector('getblock', params_iterable)
+        # Convert hex string to bytes
+        return [hex_to_bytes(self.strip_mtp_data(block)) for block in blocks]
+
+    async def masternode_broadcast(self, params):
+        '''Broadcast a transaction to the network.'''
+        return await self._send_single('znodebroadcast', params)
+
+    async def masternode_list(self, params):
+        '''Return the masternode status.'''
+        return await self._send_single('znodelist', params)
+
+    async def protx(self, params):
+        '''Set of commands to execute ProTx related actions.'''
+        return await self._send_single('protx', params)

src/electrumx/server/db.py

@@ -0,0 +1,820 @@
+# Copyright (c) 2016-2020, Neil Booth
+# Copyright (c) 2017, the ElectrumX authors
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Interface to the blockchain database.'''
+
+
+from array import array
+import ast
+import asyncio
+import os
+import time
+from bisect import bisect_right
+from dataclasses import dataclass
+from glob import glob
+from typing import Dict, List, Sequence, Tuple, Optional, TYPE_CHECKING, Union, Callable, Iterable
+
+from aiorpcx import run_in_thread, sleep
+
+import electrumx.lib.util as util
+from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN, hex_str_to_hash
+from electrumx.lib.merkle import Merkle, MerkleCache
+from electrumx.lib.util import (
+    formatted_time,
+)
+from electrumx.lib.tx import TXOSpendStatus
+from electrumx.server.storage import db_class, Storage
+from electrumx.server.history import (
+    History,
+)
+from electrumx.server.db_util import (
+    DBTooOldForMigrations,
+    pack_txnum, unpack_txnum, TXNUM_LEN,
+    pack_txoutidx, unpack_txoutidx, TXOUTIDX_LEN,
+    pack_satoshis_val, unpack_satoshis_val,
+    pack_block_height, unpack_block_height, BHEIGHT_LEN,
+    pack_dyn_header_offset, unpack_dyn_header_offset, DYN_HEADER_OFFSET_LEN,
+)
+
+if TYPE_CHECKING:
+    from electrumx.server.env import Env
+
+
+@dataclass(slots=True, order=True)
+class UTXO:
+    tx_num: int      # index of tx in chain order
+    tx_pos: int      # tx output idx
+    txid_rev: bytes | None
+    height: int      # block height
+    value: int       # in satoshis
+
+
+@dataclass(slots=True, kw_only=True)
+class FlushData:
+    height: int
+    tx_count: int
+    headers: list[bytes]
+    bhash_to_bheight: dict[bytes, int]  # block_hash_rev -> block_height
+    block_txids_rev: list[bytes]
+    # The following are flushed to the UTXO DB if undo_infos is not None
+    undo_infos: dict[int, Sequence[bytes]]  # height -> undo_info
+    adds: dict[bytes, bytes]  # txid+out_idx -> hashX+tx_num+value_sats
+    deletes: list[bytes]  # b'h' db keys, and b'u' db keys.  order does not matter
+    tip: bytes
+
+
+COMP_TXID_LEN = 4
+
+
+class DB:
+    '''Simple wrapper of the backend database for querying.
+
+    Performs no DB update, though the DB will be cleaned on opening if
+    it was shutdown uncleanly.
+    '''
+
+    DB_VERSIONS = (11, )
+    DB_STATE_KEY = b'state'
+
+    utxo_db: Optional['Storage']
+
+    class DBError(Exception):
+        '''Raised on general DB errors generally indicating corruption.'''
+
+    def __init__(self, env: 'Env'):
+        self.logger = util.class_logger(__name__, self.__class__.__name__)
+        self.env = env
+        self.coin = env.coin
+
+        # Setup block header size handlers
+        if self.coin.STATIC_BLOCK_HEADERS:
+            self.header_offset = self.coin.static_header_offset  # type: Callable[[int], int]
+            self.header_len = self.coin.static_header_len  # type: Callable[[int], int]
+        else:
+            self.header_offset = self.dynamic_header_offset  # type: Callable[[int], int]
+            self.header_len = self.dynamic_header_len  # type: Callable[[int], int]
+
+        self.logger.info(f'switching current directory to {env.db_dir}')
+        os.chdir(env.db_dir)
+
+        self.db_class = db_class(self.env.db_engine)
+        self.history = History()
+
+        # Key: b'u' + address_hashX + txout_idx + tx_num
+        # Value: the UTXO value as a 64-bit unsigned integer (in satoshis)
+        # "at address, at outpoint, there is a UTXO of value v"
+        # ---
+        # Key: b'h' + compressed_tx_hash + txout_idx + tx_num
+        # Value: hashX
+        # "some outpoint created a UTXO at address"
+        # ---
+        # Key: b'U' + block_height
+        # Value: byte-concat list of (hashX + tx_num + value_sats)
+        # "undo data: list of UTXOs spent at block height"
+        self.utxo_db = None
+
+        self.fs_height = -1
+        self.fs_tx_count = 0
+        self.db_height = -1
+        self.db_tx_count = 0
+        self.db_tip = None  # type: Optional[bytes]  # block_hash_rev
+        self.last_flush = time.time()
+        self.last_flush_tx_count = 0
+        self.wall_time = 0
+        self.first_sync = True
+        self.db_version = -1
+        self.db_flushed_event = asyncio.Event()
+
+        self.logger.info(f'using {self.env.db_engine} for DB backend')
+
+        # Header merkle cache
+        self.merkle = Merkle()
+        self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes_rev)
+
+        # note: all on-disk and in-memory dbs are consistent with each other if BlockProcessor.tip == DB.tip.
+        #       (the mempool is out-of-scope here)
+
+        # on-disk: raw block headers in chain order
+        self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
+        # on-disk: cumulative number of txs at the end of height N
+        self.tx_counts = None  # type: Optional[array]  # in-memory
+        self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
+        # on-disk: 32 byte txids in chain order, allows (tx_num -> txid_rev) map
+        self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
+        if not self.coin.STATIC_BLOCK_HEADERS:
+            self.headers_offsets_file = util.LogicalFile(
+                'meta/headers_offsets', 2, 16000000)
+
+        # in-memory: (block_hash_rev -> block_height) full map
+        self.bhash_to_bheight = None  # type: Optional[dict[bytes, int]]
+
+    async def _read_tx_counts(self) -> None:
+        if self.tx_counts is not None:
+            return
+        # tx_counts[N] has the cumulative number of txs at the end of
+        # height N.  So tx_counts[0] is 1 - the genesis coinbase
+        size = (self.db_height + 1) * 8
+        tx_counts = self.tx_counts_file.read(0, size)
+        assert len(tx_counts) == size
+        self.tx_counts = array('Q', tx_counts)
+        if self.tx_counts:
+            assert self.db_tx_count == self.tx_counts[-1]
+        else:
+            assert self.db_tx_count == 0
+
+    async def _open_dbs(self, *, for_sync: bool) -> None:
+        assert self.utxo_db is None
+
+        # First UTXO DB
+        def open_utxo_db():
+            self.utxo_db = self.db_class('utxo', for_sync)
+            if self.utxo_db.is_new:
+                self.logger.info('created new database')
+                self.logger.info('creating metadata directory')
+                os.mkdir('meta')
+                with util.open_file('COIN', create=True) as f:
+                    f.write(f'ElectrumX databases and metadata for '
+                            f'{self.coin.NAME} {self.coin.NET}'.encode())
+                if not self.coin.STATIC_BLOCK_HEADERS:
+                    self.headers_offsets_file.write(0, b'\0\0\0\0\0\0\0\0')
+            else:
+                self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
+            self.read_utxo_state()
+        await run_in_thread(open_utxo_db)
+
+        # Then history DB
+        def open_hist_db():
+            self.history.open_db(
+                db_class=self.db_class,
+                for_sync=for_sync,
+                utxo_db_tx_count=self.db_tx_count,
+            )
+            self.clear_excess_undo_info()
+        await run_in_thread(open_hist_db)
+
+        # Now prepare in-memory structures.
+        # - Read TX counts (requires meta directory)
+        await self._read_tx_counts()
+        # - (block_hash_rev -> block_height) map
+        await self._prep_bhash_to_bheight_map()
+
+    async def open_for_sync(self) -> None:
+        '''Open the databases to sync to the daemon.
+
+        When syncing we want to reserve a lot of open files for the
+        synchronization.  When serving clients we want the open files for
+        serving network connections.
+        '''
+        await self._open_dbs(for_sync=True)
+
+    async def open_for_serving(self) -> None:
+        '''Open the databases for serving.  If they are already open they are
+        closed first.
+        '''
+        if self.utxo_db:
+            self.logger.info('closing DBs to re-open for serving')
+            self.utxo_db.close()
+            self.history.close_db()
+            self.utxo_db = None
+        await self._open_dbs(for_sync=False)
+
+    # Header merkle cache
+
+    async def populate_header_merkle_cache(self) -> None:
+        self.logger.info('populating header merkle cache...')
+        length = max(1, self.db_height - self.env.reorg_limit)
+        start = time.monotonic()
+        await self.header_mc.initialize(length)
+        elapsed = time.monotonic() - start
+        self.logger.info(f'header merkle cache populated in {elapsed:.1f}s')
+
+    async def header_branch_and_root(self, length: int, height: int) -> tuple[list[bytes], bytes]:
+        return await self.header_mc.branch_and_root(length, height)
+
+    # -- (block_hash -> block_height) map
+    async def _prep_bhash_to_bheight_map(self) -> None:
+        if self.bhash_to_bheight is not None:
+            return
+        self.logger.info('preparing bhash->bheight map...')
+        t0 = time.monotonic()
+        self.bhash_to_bheight = {}
+        count = self.db_height + 1
+        block_hashes = await self.fs_block_hashes_rev(0, count)
+        if len(block_hashes) != count:
+            raise Exception(
+                f"failed to prep bhash_to_bheight. "
+                f"wanted {count} bhashes, only got {len(block_hashes)}")
+        for bheight, bhash in enumerate(block_hashes):
+            self.bhash_to_bheight[bhash] = bheight
+        assert len(self.bhash_to_bheight) == self.db_height + 1
+        self.logger.info(f'bhash->bheight map populated in {time.monotonic() - t0:.1f}s')
+
+    def get_blockheight_from_blockhash(self, block_hash_hum: str) -> Optional[int]:
+        bhash_rev = hex_str_to_hash(block_hash_hum)
+        return self.bhash_to_bheight.get(bhash_rev, None)
+
+    # Flushing
+    def assert_flushed(self, flush_data: FlushData) -> None:
+        '''Asserts state is fully flushed.'''
+        assert flush_data.tx_count == self.fs_tx_count == self.db_tx_count
+        assert flush_data.height == self.fs_height == self.db_height
+        assert flush_data.tip == self.db_tip
+        assert not flush_data.headers
+        assert not flush_data.bhash_to_bheight
+        assert not flush_data.block_txids_rev
+        assert not flush_data.adds
+        assert not flush_data.deletes
+        assert not flush_data.undo_infos
+        self.history.assert_flushed()
+
+    def flush_dbs(
+            self,
+            *,
+            flush_data: FlushData,
+            flush_utxos: bool,
+            estimate_txs_remaining: Callable[[], float],
+    ) -> None:
+        '''Flush out cached state.  History is always flushed; UTXOs are
+        flushed if flush_utxos.'''
+        if flush_data.height == self.db_height:
+            assert flush_data.tip == self.db_tip, f"block_hash mismatch at {flush_data.height}"
+            self.assert_flushed(flush_data)
+            return
+
+        start_time = time.time()
+        prior_flush = self.last_flush
+        tx_delta = flush_data.tx_count - self.last_flush_tx_count
+
+        # 1. Flush to file system
+        self._flush_fs(flush_data)
+
+        # 2. Then hist_db
+        self._flush_history()
+
+        # 3. Then utxo_db
+        # Flush state last as it reads the wall time.
+        with self.utxo_db.write_batch() as batch:
+            if flush_utxos:
+                self._flush_utxo_db(batch, flush_data)
+            self._flush_state(batch)
+
+        # Update and put the wall time again - otherwise we drop the
+        # time it took to commit the batch
+        self._flush_state(self.utxo_db)
+
+        elapsed = self.last_flush - start_time
+        self.logger.info(f'flush took '
+                         f'{elapsed:.1f}s.  Height {flush_data.height:,d} '
+                         f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
+
+        # Catch-up stats
+        if self.utxo_db.for_sync:
+            flush_interval = self.last_flush - prior_flush
+            tx_per_sec_gen = int(flush_data.tx_count / self.wall_time)
+            tx_per_sec_last = 1 + int(tx_delta / flush_interval)
+            eta = estimate_txs_remaining() / tx_per_sec_last
+            self.logger.info(f'tx/sec since genesis: {tx_per_sec_gen:,d}, '
+                             f'since last flush: {tx_per_sec_last:,d}')
+            self.logger.info(f'sync time: {formatted_time(self.wall_time)}  '
+                             f'ETA: {formatted_time(eta)}')
+
+    def _flush_fs(self, flush_data: FlushData) -> None:
+        '''Write headers, tx counts and block tx hashes to the filesystem.
+
+        The first height to write is self.fs_height + 1.  The FS
+        metadata is all append-only, so in a crash we just pick up
+        again from the height stored in the DB.
+        '''
+        prior_tx_count = (self.tx_counts[self.fs_height]
+                          if self.fs_height >= 0 else 0)
+        assert len(flush_data.block_txids_rev) == len(flush_data.headers)
+        assert len(flush_data.bhash_to_bheight) == len(flush_data.headers)
+        assert flush_data.height == self.fs_height + len(flush_data.headers)
+        assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
+                                       else 0)
+        assert len(self.tx_counts) == flush_data.height + 1
+        txids_rev = b''.join(flush_data.block_txids_rev)
+        flush_data.block_txids_rev.clear()
+        assert len(txids_rev) % 32 == 0
+        assert len(txids_rev) // 32 == flush_data.tx_count - prior_tx_count
+
+        # Write the headers, tx counts, and tx hashes
+        start_time = time.monotonic()
+        height_start = self.fs_height + 1
+        offset = self.header_offset(height_start)
+        self.headers_file.write(offset, b''.join(flush_data.headers))
+        self.fs_update_header_offsets(offset_start=offset, height_start=height_start, headers=flush_data.headers)
+        flush_data.headers.clear()
+        self.bhash_to_bheight.update(flush_data.bhash_to_bheight)
+        assert len(self.bhash_to_bheight) == flush_data.height + 1
+        flush_data.bhash_to_bheight.clear()
+
+        offset = height_start * self.tx_counts.itemsize
+        self.tx_counts_file.write(offset,
+                                  self.tx_counts[height_start:].tobytes())
+        offset = prior_tx_count * 32
+        self.hashes_file.write(offset, txids_rev)
+
+        self.fs_height = flush_data.height
+        self.fs_tx_count = flush_data.tx_count
+
+        if self.utxo_db.for_sync:
+            elapsed = time.monotonic() - start_time
+            self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
+
+    def _flush_history(self) -> None:
+        self.history.flush()
+
+    def _flush_utxo_db(self, batch, flush_data: FlushData) -> None:
+        '''Flush the cached DB writes and UTXO set to the batch.'''
+        # Care is needed because the writes generated by flushing the
+        # UTXO state may have keys in common with our write cache or
+        # may be in the DB already.
+        start_time = time.monotonic()
+        add_count = len(flush_data.adds)
+        spend_count = len(flush_data.deletes) // 2
+
+        # Spends
+        batch_delete = batch.delete
+        for key in sorted(flush_data.deletes):
+            batch_delete(key)
+        flush_data.deletes.clear()
+
+        # New UTXOs
+        batch_put = batch.put
+        for key, value in flush_data.adds.items():
+            # key: txid+out_idx, value: hashX+tx_num+value_sats
+            hashX = value[:HASHX_LEN]
+            txout_idx = key[-TXOUTIDX_LEN:]
+            tx_num = value[HASHX_LEN: HASHX_LEN+TXNUM_LEN]
+            value_sats = value[-8:]
+            suffix = txout_idx + tx_num
+            batch_put(b'h' + key[:COMP_TXID_LEN] + suffix, hashX)
+            batch_put(b'u' + hashX + suffix, value_sats)
+        flush_data.adds.clear()
+
+        # Add new undo information
+        self.flush_undo_infos(batch_put, flush_data.undo_infos)
+        flush_data.undo_infos.clear()
+        # Delete old undo information
+        if not self.utxo_db.for_sync:  # undo infos were only added if we are nearly caught up anyway
+            old_min_height = max(0, self.min_undo_height(self.db_height))
+            new_min_height = max(0, self.min_undo_height(flush_data.height))
+            for h in range(old_min_height, new_min_height):
+                key = self.undo_key(h)
+                batch_delete(key)
+
+        # Log stats
+        if self.utxo_db.for_sync:
+            block_count = flush_data.height - self.db_height
+            tx_count = flush_data.tx_count - self.db_tx_count
+            elapsed = time.monotonic() - start_time
+            self.logger.info(f'flushed {block_count:,d} blocks with '
+                             f'{tx_count:,d} txs, {add_count:,d} UTXO adds, '
+                             f'{spend_count:,d} spends in '
+                             f'{elapsed:.1f}s, committing...')
+
+        self.db_height = flush_data.height
+        self.db_tx_count = flush_data.tx_count
+        self.db_tip = flush_data.tip
+
+    def _flush_state(self, batch):
+        '''Flush chain state to the batch.'''
+        now = time.time()
+        self.wall_time += now - self.last_flush
+        self.last_flush = now
+        self.last_flush_tx_count = self.fs_tx_count
+        self.write_utxo_state(batch)
+
+    def flush_backup(self, flush_data: FlushData, touched_hashxs: Iterable[bytes]) -> None:
+        '''Like flush_dbs() but when backing up.  All UTXOs are flushed.'''
+        assert not flush_data.headers
+        assert not flush_data.block_txids_rev
+        assert flush_data.height < self.db_height
+        assert len(flush_data.bhash_to_bheight) == (self.db_height - flush_data.height)
+        self.history.assert_flushed()
+
+        start_time = time.time()
+        tx_delta = flush_data.tx_count - self.last_flush_tx_count
+
+        # 1. (fake-)Flush to file system: this just updates pointers
+        self._backup_fs(flush_data)
+
+        # Crucially, we flush utxo_db and hist_db in the reverse order compared to when advancing,
+        #   to maintain the invariant that hist_db_tx_count >= utxo_db_tx_count even if we crash.
+        # 2. Then utxo_db
+        with self.utxo_db.write_batch() as batch:
+            self._flush_utxo_db(batch, flush_data)
+            # Flush state last as it reads the wall time.
+            self._flush_state(batch)
+
+        # 3. Then hist_db
+        self.history.backup(
+            hashXs=touched_hashxs,
+            tx_count=flush_data.tx_count,
+        )
+
+        # Update and put the wall time again - otherwise we drop the
+        # time it took to commit the batch
+        self._flush_state(self.utxo_db)
+
+        elapsed = self.last_flush - start_time
+        self.logger.info(f'backup flush took '
+                         f'{elapsed:.1f}s.  Height {flush_data.height:,d} '
+                         f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
+
+    def fs_update_header_offsets(self, *, offset_start: int, height_start: int, headers: Sequence[bytes]) -> None:
+        if self.coin.STATIC_BLOCK_HEADERS:
+            return
+        offset = offset_start
+        offsets = []
+        for h in headers:
+            offset += len(h)
+            offsets.append(pack_dyn_header_offset(offset))
+        # For each header we get the offset of the next header, hence we
+        # start writing from the next height
+        pos = (height_start + 1) * DYN_HEADER_OFFSET_LEN
+        self.headers_offsets_file.write(pos, b''.join(offsets))
+
+    def dynamic_header_offset(self, height: int) -> int:
+        assert not self.coin.STATIC_BLOCK_HEADERS
+        offset = unpack_dyn_header_offset(self.headers_offsets_file.read(height * DYN_HEADER_OFFSET_LEN, DYN_HEADER_OFFSET_LEN))
+        return offset
+
+    def dynamic_header_len(self, height: int) -> int:
+        return self.dynamic_header_offset(height + 1)\
+               - self.dynamic_header_offset(height)
+
+    def _backup_fs(self, flush_data: FlushData) -> None:
+        '''Back up during a reorg.  This just updates our pointers.'''
+        self.fs_height = flush_data.height
+        self.fs_tx_count = flush_data.tx_count
+
+        for bhash, bheight in flush_data.bhash_to_bheight.items():
+            assert bheight == self.bhash_to_bheight.pop(bhash)
+        flush_data.bhash_to_bheight.clear()
+        assert len(self.bhash_to_bheight) == flush_data.height + 1
+
+        # Truncate header_mc: header count is 1 more than the height.
+        self.header_mc.truncate(flush_data.height + 1)
+
+    async def raw_header(self, height: int) -> bytes:
+        '''Return the binary header at the given height.'''
+        header, n = await self.read_headers(height, 1)
+        if n != 1:
+            raise IndexError(f'height {height:,d} out of range')
+        return header
+
+    async def read_headers(self, start_height: int, count: int) -> Tuple[bytes, int]:
+        '''Requires start_height >= 0, count >= 0.  Reads as many headers as
+        are available starting at start_height up to count.  This
+        would be zero if start_height is beyond self.db_height, for
+        example.
+
+        Returns a (binary, n) pair where binary is the concatenated
+        binary headers, and n is the count of headers returned.
+        '''
+        if start_height < 0 or count < 0:
+            raise self.DBError(f'{count:,d} headers starting at '
+                               f'{start_height:,d} not on disk')
+
+        def read_headers():
+            # Read some from disk
+            disk_count = max(0, min(count, self.db_height + 1 - start_height))
+            if disk_count:
+                offset = self.header_offset(start_height)
+                size = self.header_offset(start_height + disk_count) - offset
+                return self.headers_file.read(offset, size), disk_count
+            return b'', 0
+
+        return await run_in_thread(read_headers)
+
+    def fs_txid_rev(self, tx_num: int) -> Tuple[Optional[bytes], int]:
+        '''Return a pair (txid_rev, tx_height) for the given tx number.
+
+        If the tx_height is not on disk, returns (None, tx_height).'''
+        tx_height = bisect_right(self.tx_counts, tx_num)
+        if tx_height > self.db_height:
+            txid_rev = None
+        else:
+            txid_rev = self.hashes_file.read(tx_num * 32, 32)
+        return txid_rev, tx_height
+
+    def fs_txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]:
+        '''Return a list of txids_rev at given block height,
+        in the same order as in the block.
+        '''
+        if block_height > self.db_height:
+            raise self.DBError(f'block {block_height:,d} not on disk (>{self.db_height:,d})')
+        assert block_height >= 0
+        if block_height > 0:
+            first_tx_num = self.tx_counts[block_height - 1]
+        else:
+            first_tx_num = 0
+        num_txs_in_block = self.tx_counts[block_height] - first_tx_num
+        txids_rev = self.hashes_file.read(first_tx_num * 32, num_txs_in_block * 32)
+        assert num_txs_in_block == len(txids_rev) // 32
+        return [txids_rev[idx * 32: (idx+1) * 32] for idx in range(num_txs_in_block)]
+
+    async def txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]:
+        return await run_in_thread(self.fs_txids_rev_at_blockheight, block_height)
+
+    async def fs_block_hashes_rev(self, height: int, count: int) -> Sequence[bytes]:
+        headers_concat, headers_count = await self.read_headers(height, count)
+        if headers_count != count:
+            raise self.DBError(f'only got {headers_count:,d} headers starting '
+                               f'at {height:,d}, not {count:,d}')
+
+        def hash_headers():
+            offset = 0
+            headers = []
+            for n in range(count):
+                hlen = self.header_len(height + n)
+                headers.append(headers_concat[offset:offset + hlen])
+                offset += hlen
+            return [self.coin.header_hash_rev(header) for header in headers]
+
+        bhashes = (await run_in_thread(hash_headers)) if count > 100 else hash_headers()
+        return bhashes
+
+    async def limited_history(self, hashX: bytes, *, limit: int = 1000) -> Sequence[tuple[bytes, int]]:
+        '''Return an unpruned, sorted list of (txid_rev, height) tuples of
+        confirmed transactions that touched the address, earliest in
+        the blockchain first.  Includes both spending and receiving
+        transactions.  By default returns at most 1000 entries.  Set
+        limit to None to get them all.
+        '''
+        def read_history():
+            tx_nums = list(self.history.get_txnums(hashX, limit))
+            fs_txid_rev = self.fs_txid_rev
+            return [fs_txid_rev(tx_num) for tx_num in tx_nums]
+
+        while True:
+            history = await run_in_thread(read_history)
+            if all(txid_rev is not None for txid_rev, height in history):
+                return history
+            self.logger.warning(f'limited_history: tx hash '
+                                f'not found (reorg?), retrying...')
+            await sleep(0.25)
+
+    # -- Undo information
+
+    def min_undo_height(self, max_height: int) -> int:
+        '''Returns a height from which we should store undo info.'''
+        return max_height - self.env.reorg_limit + 1
+
+    def undo_key(self, height: int) -> bytes:
+        '''DB key for undo information at the given height.'''
+        return b'U' + pack_block_height(height)
+
+    def read_undo_info(self, height: int) -> bytes:
+        '''Read undo information from a file for the current height.'''
+        return self.utxo_db.get(self.undo_key(height))
+
+    def flush_undo_infos(
+            self, batch_put, undo_infos: dict[int, Sequence[bytes]],
+    ) -> None:
+        '''undo_infos is a (height -> undo_info) map.'''
+        for height, undo_info in undo_infos.items():
+            batch_put(self.undo_key(height), b''.join(undo_info))
+
+    def raw_block_prefix(self) -> str:
+        return 'meta/block'
+
+    def raw_block_path(self, height: int) -> str:
+        return f'{self.raw_block_prefix()}{height:d}'
+
+    def read_raw_block(self, height: int) -> bytes:
+        '''Returns a raw block read from disk.  Raises FileNotFoundError
+        if the block isn't on-disk.'''
+        with util.open_file(self.raw_block_path(height)) as f:
+            return f.read(-1)
+
+    def write_raw_block(self, block: bytes, height: int) -> None:
+        '''Write a raw block to disk.'''
+        with util.open_truncate(self.raw_block_path(height)) as f:
+            f.write(block)
+        # Delete old blocks to prevent them accumulating
+        del_height = self.min_undo_height(height) - 1
+        if del_height >= 0:
+            try:
+                os.remove(self.raw_block_path(del_height))
+            except FileNotFoundError:
+                pass
+
+    def clear_excess_undo_info(self) -> None:
+        '''Clear excess undo info.  Only most recent N are kept.'''
+        # delete aged undo_infos from utxo db
+        # note: this is just a fallback cleanup path - normally already deleted by _flush_utxo_db()
+        min_height = self.min_undo_height(self.db_height)
+        keys = []
+        for key, uhist in self.utxo_db.iterator(prefix=b'U'):
+            height = unpack_block_height(key[-BHEIGHT_LEN:])
+            if height >= min_height:
+                break  # can break as block_height is encoded as big endian
+            keys.append(key)
+
+        if keys:
+            with self.utxo_db.write_batch() as batch:
+                for key in keys:
+                    batch.delete(key)
+            self.logger.info(f'deleted {len(keys):,d} stale undo entries')
+
+        # delete old block files
+        # note: this is just a fallback cleanup path - normally already deleted by write_raw_block()
+        prefix = self.raw_block_prefix()
+        paths = [path for path in glob(f'{prefix}[0-9]*')
+                 if len(path) > len(prefix)
+                 and int(path[len(prefix):]) < min_height]
+        if paths:
+            for path in paths:
+                try:
+                    os.remove(path)
+                except FileNotFoundError:
+                    pass
+            self.logger.info(f'deleted {len(paths):,d} stale block files')
+
+    # -- UTXO database
+
+    def read_utxo_state(self) -> None:
+        state = self.utxo_db.get(self.DB_STATE_KEY)
+        if not state:
+            self.db_height = -1
+            self.db_tx_count = 0
+            self.db_tip = b'\0' * 32
+            self.db_version = max(self.DB_VERSIONS)
+            self.wall_time = 0
+            self.first_sync = True
+        else:
+            state = ast.literal_eval(state.decode())
+            if not isinstance(state, dict):
+                raise self.DBError('failed reading state from DB')
+            self.db_version = state['db_version']
+            if self.db_version not in self.DB_VERSIONS:
+                raise DBTooOldForMigrations(
+                    db_name="UTXO", db_version=self.db_version, supported_versions=self.DB_VERSIONS)
+            genesis_hash = state['genesis']
+            if genesis_hash != self.coin.GENESIS_HASH:
+                raise self.DBError(f'DB genesis hash {genesis_hash} does not '
+                                   f'match coin {self.coin.GENESIS_HASH}')
+            self.db_height = state['height']
+            self.db_tx_count = state['tx_count']
+            self.db_tip = state['tip']
+            self.wall_time = state['wall_time']
+            self.first_sync = state['first_sync']
+
+        # These are our state as we move ahead of DB state
+        self.fs_height = self.db_height
+        self.fs_tx_count = self.db_tx_count
+        self.last_flush_tx_count = self.fs_tx_count
+
+        # Upgrade DB
+        if self.db_version != max(self.DB_VERSIONS):
+            raise Exception("missing db upgrade")  # call future upgrade logic here
+
+        # Log some stats
+        self.logger.info(f'UTXO DB version: {self.db_version:d}')
+        self.logger.info(f'coin: {self.coin.NAME}')
+        self.logger.info(f'network: {self.coin.NET}')
+        self.logger.info(f'height: {self.db_height:,d}')
+        self.logger.info(f'tip: {hash_to_hex_str(self.db_tip)}')
+        self.logger.info(f'tx count: {self.db_tx_count:,d}')
+        if self.utxo_db.for_sync:
+            self.logger.info(f'flushing DB cache at {self.env.cache_MB:,d} MB')
+        if self.first_sync:
+            self.logger.info(
+                f'sync time so far: {util.formatted_time(self.wall_time)}'
+            )
+
+    def write_utxo_state(self, batch) -> None:
+        '''Write (UTXO) state to the batch.'''
+        state = {
+            'genesis': self.coin.GENESIS_HASH,
+            'height': self.db_height,
+            'tx_count': self.db_tx_count,
+            'tip': self.db_tip,
+            'wall_time': self.wall_time,
+            'first_sync': self.first_sync,
+            'db_version': self.db_version,
+        }
+        batch.put(self.DB_STATE_KEY, repr(state).encode())
+
+    async def all_utxos(self, hashX: bytes) -> Sequence[UTXO]:
+        '''Return all UTXOs for an address sorted in no particular order.'''
+        def read_utxos():
+            utxos = []  # type: list[UTXO]
+            utxos_append = utxos.append
+            # Key: b'u' + address_hashX + txout_idx + tx_num
+            # Value: the UTXO value as a 64-bit unsigned integer
+            prefix = b'u' + hashX
+            for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
+                txout_idx = unpack_txoutidx(db_key[-TXNUM_LEN-TXOUTIDX_LEN:-TXNUM_LEN])
+                tx_num = unpack_txnum(db_key[-TXNUM_LEN:])
+                value = unpack_satoshis_val(db_value)
+                txid_rev, height = self.fs_txid_rev(tx_num)
+                utxos_append(UTXO(tx_num, txout_idx, txid_rev, height, value))
+            return utxos
+
+        while True:
+            utxos = await run_in_thread(read_utxos)
+            if all(utxo.txid_rev is not None for utxo in utxos):
+                return utxos
+            self.logger.warning(f'all_utxos: tx hash not '
+                                f'found (reorg?), retrying...')
+            await sleep(0.25)
+
+    async def lookup_utxos(self, prevouts: Sequence[tuple[bytes, int]]) -> Sequence[Optional[tuple[bytes, int]]]:
+        '''For each prevout, lookup it up in the DB and return a (hashX,
+        value) pair or None if not found.
+
+        Used by the mempool code.
+        '''
+        def lookup_hashXs() -> Sequence[tuple[Optional[bytes], Optional[bytes]]]:
+            '''Return (hashX, suffix) pairs, or None if not found,
+            for each prevout.
+            '''
+            def lookup_hashX(txid_rev: bytes, tx_idx: int) -> tuple[Optional[bytes], Optional[bytes]]:
+                idx_packed = pack_txoutidx(tx_idx)
+
+                # Key: b'h' + compressed_tx_hash + tx_idx + tx_num
+                # Value: hashX
+                prefix = b'h' + txid_rev[:COMP_TXID_LEN] + idx_packed
+
+                # Find which entry, if any, the TX_HASH matches.
+                for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
+                    tx_num_packed = db_key[-TXNUM_LEN:]
+                    tx_num = unpack_txnum(tx_num_packed)
+                    hash, _height = self.fs_txid_rev(tx_num)
+                    if hash == txid_rev:
+                        return hashX, idx_packed + tx_num_packed
+                return None, None
+            return [lookup_hashX(*prevout) for prevout in prevouts]
+
+        def lookup_utxos(
+                hashX_pairs: Sequence[tuple[Optional[bytes], Optional[bytes]]],
+        ) -> Sequence[Optional[tuple[bytes, int]]]:
+            def lookup_utxo(hashX: bytes, suffix: bytes) -> Optional[tuple[bytes, int]]:
+                if not hashX:
+                    # This can happen when the daemon is a block ahead
+                    # of us and has mempool txs spending outputs from
+                    # that new block
+                    return None
+                # Key: b'u' + address_hashX + tx_idx + tx_num
+                # Value: the UTXO value as a 64-bit unsigned integer
+                key = b'u' + hashX + suffix
+                db_value = self.utxo_db.get(key)
+                if not db_value:
+                    # This can happen if the DB was updated between
+                    # getting the hashXs and getting the UTXOs
+                    return None
+                value = unpack_satoshis_val(db_value)
+                return hashX, value
+            return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
+
+        hashX_pairs = await run_in_thread(lookup_hashXs)
+        return await run_in_thread(lookup_utxos, hashX_pairs)

src/electrumx/server/db_util.py

@@ -0,0 +1,88 @@
+# Copyright (C) 2026 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from typing import Sequence
+
+from electrumx.lib.util import (
+    pack_be_uint32, unpack_be_uint32,
+    pack_be_uint64, unpack_be_uint64,
+)
+
+
+# txnum
+TXNUM_LEN = 5
+TXNUM_PADDING = bytes(8 - TXNUM_LEN)
+
+
+def unpack_txnum(tx_numb: bytes) -> int:
+    return unpack_be_uint64(TXNUM_PADDING + tx_numb)[0]
+
+
+def pack_txnum(tx_num: int) -> bytes:
+    return pack_be_uint64(tx_num)[-TXNUM_LEN:]
+
+
+assert pack_txnum(0x123) == bytes.fromhex("0000000123")
+assert unpack_txnum(pack_txnum(0x123)) == 0x123
+
+
+# txout_idx
+TXOUTIDX_LEN = 3
+TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN)
+
+
+def unpack_txoutidx(txout_idx: bytes) -> int:
+    return unpack_be_uint32(TXOUTIDX_PADDING + txout_idx)[0]
+
+
+def pack_txoutidx(txout_idx: int) -> bytes:
+    return pack_be_uint32(txout_idx)[-TXOUTIDX_LEN:]
+
+
+assert pack_txoutidx(0x123) == bytes.fromhex("000123")
+assert unpack_txoutidx(pack_txoutidx(0x123)) == 0x123
+
+
+# sats
+def unpack_satoshis_val(sats: bytes) -> int:
+    return unpack_be_uint64(sats)[0]
+
+
+def pack_satoshis_val(sats: int) -> bytes:
+    return pack_be_uint64(sats)
+
+
+# block height
+BHEIGHT_LEN = 4
+
+
+def unpack_block_height(bheight: bytes) -> int:
+    return unpack_be_uint32(bheight)[0]
+
+
+def pack_block_height(bheight: int) -> bytes:
+    return pack_be_uint32(bheight)
+
+
+# dynamic block header offsets
+DYN_HEADER_OFFSET_LEN = 8
+
+
+def unpack_dyn_header_offset(offset: bytes) -> int:
+    return unpack_be_uint64(offset)[0]
+
+
+def pack_dyn_header_offset(offset: int) -> bytes:
+    return pack_be_uint64(offset)
+
+
+class DBTooOldForMigrations(RuntimeError):
+    def __init__(self, *, db_name: str, db_version: int, supported_versions: Sequence[int]):
+        cmd = 'rm -rf DB_DIRECTORY/{hist,meta,utxo}'
+        super().__init__(
+            f'Your {db_name} DB version is {db_version} but this software only handles versions {supported_versions}. '
+            f'Manually delete your database (e.g. `{cmd}`, and start again. '
+            f'Then, your DB will be rebuilt from genesis, likely taking several hours. '
+            f"If you don't have time for this now, you can temporarily downgrade the software."
+        )

src/electrumx/server/env.py

@@ -0,0 +1,245 @@
+# Copyright (c) 2016, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Class for handling environment configuration and defaults.'''
+
+
+import re
+from ipaddress import IPv4Address, IPv6Address
+from typing import Type, Sequence
+
+from aiorpcx import Service, ServicePart
+
+import electrumx
+from electrumx.lib.coins import Coin
+from electrumx.lib.env_base import EnvBase
+
+
+class ServiceError(Exception):
+    pass
+
+
+class Env(EnvBase):
+    '''Wraps environment configuration. Optionally, accepts a Coin class
+       as first argument to have ElectrumX serve custom coins not part of
+       the standard distribution.
+    '''
+
+    # Peer discovery
+    PD_OFF, PD_SELF, PD_ON = ('OFF', 'SELF', 'ON')
+    SSL_PROTOCOLS = {'ssl', 'wss'}
+    KNOWN_PROTOCOLS = {'ssl', 'tcp', 'ws', 'wss', 'rpc'}
+
+    coin: Type[Coin]
+
+    def __init__(self, coin=None):
+        super().__init__()
+        self.obsolete(["MAX_SUBSCRIPTIONS", "MAX_SUBS", "MAX_SESSION_SUBS", "BANDWIDTH_LIMIT",
+                       "HOST", "TCP_PORT", "SSL_PORT", "RPC_HOST", "RPC_PORT", "REPORT_HOST",
+                       "REPORT_TCP_PORT", "REPORT_SSL_PORT", "REPORT_HOST_TOR",
+                       "REPORT_TCP_PORT_TOR", "REPORT_SSL_PORT_TOR"])
+
+        # Core items
+
+        self.db_dir = self.required('DB_DIRECTORY')
+        self.db_engine = self.db_engine_enum()
+        self.daemon_url = self.required('DAEMON_URL')
+        if coin is not None:
+            assert issubclass(coin, Coin)
+            self.coin = coin
+        else:
+            coin_name = self.default('COIN', 'Bitcoin').strip()
+            network = self.default('NET', 'mainnet').strip()
+            self.coin = Coin.lookup_coin_class(coin_name, network)
+
+        # Peer discovery
+
+        self.peer_discovery = self.peer_discovery_enum()
+        self.peer_announce = self.boolean('PEER_ANNOUNCE', True)
+        self.force_proxy = self.boolean('FORCE_PROXY', False)
+        self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
+        self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
+
+        # Misc
+        self.banner_file = self.default('BANNER_FILE', None)
+        self.tor_banner_file = self.default('TOR_BANNER_FILE',
+                                            self.banner_file)
+        self.anon_logs = self.boolean('ANON_LOGS', False)
+        self.log_sessions = self.integer('LOG_SESSIONS', 3600)
+        self.log_level = self.default('LOG_LEVEL', 'info').upper()
+        self.donation_address = self.default('DONATION_ADDRESS', '')
+        self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
+        self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
+        self.cache_MB = self.integer('CACHE_MB', 1200)
+        self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
+        self.daemon_poll_interval_blocks_msec = self.integer('DAEMON_POLL_INTERVAL_BLOCKS', 5000)
+        self.daemon_poll_interval_mempool_msec = self.integer('DAEMON_POLL_INTERVAL_MEMPOOL', 5000)
+
+        # Server limits to help prevent DoS
+
+        self.max_send = self.integer('MAX_SEND', self.coin.DEFAULT_MAX_SEND)
+        self.max_recv = self.integer('MAX_RECV', self.coin.DEFAULT_MAX_RECV)
+        self._try_raising_rlimit_nofile()  # do this before setting max_sessions
+        self.max_sessions = self.sane_max_sessions()
+        self.cost_soft_limit = self.integer('COST_SOFT_LIMIT', 1000)
+        self.cost_hard_limit = self.integer('COST_HARD_LIMIT', 10000)
+        self.bw_unit_cost = self.integer('BANDWIDTH_UNIT_COST', 5000)
+        self.initial_concurrent = self.integer('INITIAL_CONCURRENT', 10)
+        self.request_sleep = self.integer('REQUEST_SLEEP', 2500)
+        self.request_timeout = self.integer('REQUEST_TIMEOUT', 30)
+        self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
+        self.session_group_by_subnet_ipv4 = self.integer('SESSION_GROUP_BY_SUBNET_IPV4', 24)
+        self.session_group_by_subnet_ipv6 = self.integer('SESSION_GROUP_BY_SUBNET_IPV6', 48)
+        self._check_and_fix_cost_limits()
+
+        # Services last - uses some env vars above
+
+        self.services = self.services_to_run()
+        if {service.protocol for service in self.services}.intersection(self.SSL_PROTOCOLS):
+            self.ssl_certfile = self.required('SSL_CERTFILE')
+            self.ssl_keyfile = self.required('SSL_KEYFILE')
+        self.report_services = self.services_to_report()
+
+    def _try_raising_rlimit_nofile(self) -> None:
+        """Try raising the max num open file soft limit to the hard limit."""
+        try:
+            import resource
+        except ImportError:
+            return  # No resource module on Windows
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if soft >= hard:
+            return
+        try:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+        except (ValueError, OSError):
+            self.logger.warning(
+                f"error raising RLIMIT_NOFILE (ulimit -n) soft limit. Stuck with {soft}. "
+                f"You must ensure that electrumx has a large open file limit. See HOWTO in docs.")
+        else:
+            self.logger.info(
+                f"successfully raised RLIMIT_NOFILE (ulimit -n) soft limit from {soft} to {hard}")
+
+    def sane_max_sessions(self):
+        '''Return the maximum number of sessions to permit.  Normally this
+        is MAX_SESSIONS.  However, to prevent open file exhaustion, adjust
+        downwards if running with a small open file rlimit.'''
+        env_value = self.integer('MAX_SESSIONS', 1000)
+        # No resource module on Windows
+        try:
+            import resource
+            nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
+            # We give the DB 250 files; allow ElectrumX 100 for itself
+            value = max(0, min(env_value, nofile_limit - 350))
+            if value < env_value:
+                self.logger.warning(
+                    f'lowered maximum sessions from {env_value:,d} to '
+                    f'{value:,d} because your open file limit is '
+                    f'{nofile_limit:,d}'
+                )
+        except ImportError:
+            value = 512  # that is what returned by stdio's _getmaxstdio()
+        return value
+
+    def _check_and_fix_cost_limits(self):
+        if self.cost_hard_limit < self.cost_soft_limit:
+            raise self.Error(f"COST_HARD_LIMIT must be >= COST_SOFT_LIMIT. "
+                             f"got (COST_HARD_LIMIT={self.cost_hard_limit} "
+                             f"and COST_SOFT_LIMIT={self.cost_soft_limit})")
+        # hard limit should be strictly higher than soft limit (unless both are 0)
+        if self.cost_hard_limit == self.cost_soft_limit and self.cost_soft_limit > 0:
+            self.logger.info("found COST_HARD_LIMIT == COST_SOFT_LIMIT. "
+                             "bumping COST_HARD_LIMIT by 1.")
+            self.cost_hard_limit = self.cost_soft_limit + 1
+
+    def _parse_services(self, services_str, default_func) -> Sequence[Service]:
+        result = []
+        for service_str in services_str.split(','):
+            if not service_str:
+                continue
+            try:
+                service = Service.from_string(service_str, default_func=default_func)
+            except Exception as e:
+                raise ServiceError(f'"{service_str}" invalid: {e}') from None
+            if service.protocol not in self.KNOWN_PROTOCOLS:
+                raise ServiceError(f'"{service_str}" invalid: unknown protocol')
+            result.append(service)
+
+        # Find duplicate addresses
+        service_map = {service.address: [] for service in result}
+        for service in result:
+            service_map[service.address].append(service)
+        for address, services in service_map.items():
+            if len(services) > 1:
+                raise ServiceError(f'address {address} has multiple services')
+
+        return result
+
+    def services_to_run(self) -> Sequence[Service]:
+        def default_part(protocol, part):
+            return default_services.get(protocol, {}).get(part)
+
+        default_services = {protocol: {ServicePart.HOST: 'all_interfaces'}
+                            for protocol in self.KNOWN_PROTOCOLS}
+        default_services['rpc'] = {ServicePart.HOST: 'localhost', ServicePart.PORT: 8000}
+        services = self._parse_services(self.default('SERVICES', ''), default_part)
+
+        # Find onion hosts
+        for service in services:
+            if str(service.host).endswith('.onion'):
+                raise ServiceError(f'bad host for SERVICES: {service}')
+
+        return services
+
+    def services_to_report(self):
+        services = self._parse_services(self.default('REPORT_SERVICES', ''), None)
+
+        for service in services:
+            if service.protocol == 'rpc':
+                raise ServiceError(f'bad protocol for REPORT_SERVICES: {service.protocol}')
+            if isinstance(service.host, (IPv4Address, IPv6Address)):
+                ip_addr = service.host
+                if (ip_addr.is_multicast or ip_addr.is_unspecified or
+                        (ip_addr.is_private and self.peer_announce)):
+                    raise ServiceError(f'bad IP address for REPORT_SERVICES: {ip_addr}')
+            elif service.host.lower() == 'localhost':
+                raise ServiceError(f'bad host for REPORT_SERVICES: {service.host}')
+
+        return services
+
+    def peer_discovery_enum(self):
+        pd = self.default('PEER_DISCOVERY', 'on').strip().lower()
+        if pd in ('off', ''):
+            return self.PD_OFF
+        elif pd == 'self':
+            return self.PD_SELF
+        else:
+            return self.PD_ON
+
+    def db_engine_enum(self) -> str:
+        from electrumx.server.storage import list_db_engine_choices
+        choices = list_db_engine_choices()
+        db_eng = self.default('DB_ENGINE', None)
+
+        if db_eng is None:
+            raise self.Error(
+                f"required envvar DB_ENGINE not set.\n"
+                f"Choose one from: {choices}.\n"
+                f"The corresponding dependencies also need to be installed. See pip extras.\n"
+                f"In ElectrumX 1.x versions, the default was leveldb.\n"
+                f"Warning: It is not possible to switch back and forth, "
+                f"the on-disk DB formats are not compatible with each other: when switching, "
+                f"you have to resync from genesis.\n"
+                f"LevelDB was written with HDDs in mind. RocksDB is more modern and performs better (on an SSD)."
+            )
+
+        db_eng = db_eng.lower()
+        if db_eng in choices:
+            return db_eng
+        else:
+            raise self.Error(
+                f"bad value for envvar DB_ENGINE: {db_eng!r}. Choose one from: {choices}."
+            )

src/electrumx/server/history.py

@@ -0,0 +1,210 @@
+# Copyright (c) 2016-2018, Neil Booth
+# Copyright (c) 2017, the ElectrumX authors
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''History by script hash (address).'''
+
+import ast
+import bisect
+import time
+from array import array
+from collections import defaultdict
+from typing import TYPE_CHECKING, Type, Optional, Sequence, Iterable
+
+import electrumx.lib.util as util
+from electrumx.lib.hash import HASHX_LEN
+from electrumx.server.db_util import (
+    DBTooOldForMigrations,
+    pack_txnum, unpack_txnum, TXNUM_LEN,
+)
+
+if TYPE_CHECKING:
+    from electrumx.server.storage import Storage
+
+
+class History:
+
+    DB_VERSIONS = (4, )
+    DB_STATE_KEY = b'state\0\0'
+
+    db: Optional['Storage']
+
+    def __init__(self):
+        self.logger = util.class_logger(__name__, self.__class__.__name__)
+        self.unflushed = set()  # type: set[bytes]
+        self.hist_db_tx_count = 0
+        self.hist_db_tx_count_next = 0  # after next flush, next value for self.hist_db_tx_count
+        self.db_version = max(self.DB_VERSIONS)
+        self.upgrade_cursor = -1
+
+        # Key: b'H' + address_hashX + tx_num
+        # Value: <null>
+        self.db = None
+
+    def open_db(
+            self,
+            *,
+            db_class: Type['Storage'],
+            for_sync: bool,
+            utxo_db_tx_count: int,
+    ) -> None:
+        self.db = db_class('hist', for_sync)
+        self.read_state()
+        self.clear_excess(utxo_db_tx_count)
+
+    def close_db(self) -> None:
+        if self.db:
+            self.db.close()
+            self.db = None
+
+    def read_state(self) -> None:
+        state = self.db.get(self.DB_STATE_KEY)
+        if state:
+            state = ast.literal_eval(state.decode())
+            if not isinstance(state, dict):
+                raise RuntimeError('failed reading state from history DB')
+            self.db_version = state.get('db_version', 0)
+            self.upgrade_cursor = state.get('upgrade_cursor', -1)
+            self.hist_db_tx_count = state.get('hist_db_tx_count', 0)
+            self.hist_db_tx_count_next = self.hist_db_tx_count
+
+        if self.db_version not in self.DB_VERSIONS:
+            raise DBTooOldForMigrations(
+                db_name="history", db_version=self.db_version, supported_versions=self.DB_VERSIONS)
+        if self.db_version != max(self.DB_VERSIONS):
+            raise Exception("missing db upgrade")  # call future upgrade logic here
+        self.logger.info(f'history DB version: {self.db_version}')
+
+    def clear_excess(self, utxo_db_tx_count: int) -> None:
+        # self.hist_db_tx_count != utxo_db_tx_count might happen as
+        # both DBs cannot be updated atomically.
+        # However, we MUST maintain the invariant that hist_db_tx_count >= utxo_db_tx_count:
+        # - when advancing blocks, hist_db is flushed first, so its count can be higher;
+        #   but when backing up (during reorg), hist_db is flushed last, so its count can be higher.
+        if self.hist_db_tx_count == utxo_db_tx_count:
+            return  # happy path
+        elif self.hist_db_tx_count < utxo_db_tx_count:
+            raise RuntimeError(f"corrupted DB. {self.hist_db_tx_count=} < {utxo_db_tx_count=}")
+
+        assert self.hist_db_tx_count > utxo_db_tx_count
+        self.logger.info(
+            'DB shut down uncleanly. Scanning for excess history flushes... '
+            'This might take a loooong time. :/'
+        )
+
+        utxo_db_tx_count_bytes = pack_txnum(utxo_db_tx_count)
+        keys = []  # FIXME could grow large in-memory?
+        for iter_cnt, (db_key, _db_val) in enumerate(self.db.iterator(prefix=b'H')):
+            if iter_cnt % 10_000_000 == 0:
+                self.logger.info(
+                    f'Scanning for excess history flushes... num history entries checked: '
+                    f'{iter_cnt // 1_000_000} million. will delete: {len(keys):,d}')
+            tx_numb = db_key[-TXNUM_LEN:]
+            # we exploit that pack_txnum uses big-endian byte-order, and just compare the bytes directly:
+            # tx_num = unpack_txnum(tx_numb)
+            if tx_numb >= utxo_db_tx_count_bytes:
+                keys.append(db_key)
+
+        self.logger.info(f'deleting {len(keys):,d} history entries')
+
+        self.hist_db_tx_count = utxo_db_tx_count
+        self.hist_db_tx_count_next = self.hist_db_tx_count
+        with self.db.write_batch() as batch:
+            for key in keys:
+                batch.delete(key)
+            self._write_state(batch)
+
+        self.logger.info('deleted excess history entries')
+
+    def _write_state(self, batch) -> None:
+        '''Write state to the history DB.'''
+        state = {
+            'hist_db_tx_count': self.hist_db_tx_count,
+            'db_version': self.db_version,
+            'upgrade_cursor': self.upgrade_cursor,
+        }
+        batch.put(self.DB_STATE_KEY, repr(state).encode())
+
+    def add_unflushed(
+            self,
+            hashXs_by_tx: list[list[bytes]],
+            first_tx_num: int,
+    ) -> None:
+        """This method must be thread-safe. (runs on bp.pool_executor)"""
+        unflushed = self.unflushed
+        for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num):
+            tx_numb = pack_txnum(tx_num)
+            for hashX in hashXs:
+                unflushed.add(hashX + tx_numb)
+
+    def update_tx_count_next(self, tx_count: int) -> None:
+        self.hist_db_tx_count_next = tx_count
+
+    def unflushed_memsize(self) -> int:
+        # note: the magic numbers here were estimated using util.deep_getsizeof
+        return len(self.unflushed) * 85 + len(self.unflushed) * (HASHX_LEN + TXNUM_LEN)
+
+    def assert_flushed(self) -> None:
+        assert not self.unflushed
+
+    def flush(self) -> None:
+        start_time = time.monotonic()
+        unflushed = self.unflushed
+
+        with self.db.write_batch() as batch:
+            for hashX_txnum in sorted(unflushed):
+                db_key = b'H' + hashX_txnum
+                batch.put(db_key, b'')
+            self.hist_db_tx_count = self.hist_db_tx_count_next
+            self._write_state(batch)
+
+        count = len(unflushed)
+        unflushed.clear()
+
+        if self.db.for_sync:
+            elapsed = time.monotonic() - start_time
+            self.logger.info(f'flushed history in {elapsed:.1f}s '
+                             f'for {count:,d} history entries')
+
+    def backup(self, *, hashXs: Iterable[bytes], tx_count: int) -> None:
+        self.assert_flushed()
+        nremoves = 0
+        with self.db.write_batch() as batch:
+            for hashX in sorted(hashXs):
+                deletes = []
+                prefix = b'H' + hashX
+                for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True):
+                    tx_numb = db_key[-TXNUM_LEN:]
+                    tx_num = unpack_txnum(tx_numb)
+                    if tx_num >= tx_count:
+                        nremoves += 1
+                        deletes.append(db_key)
+                    else:
+                        # note: we can break now, due to 'reverse=True' and txnums being big endian
+                        break
+                for key in deletes:
+                    batch.delete(key)
+            self.hist_db_tx_count = tx_count
+            self.hist_db_tx_count_next = self.hist_db_tx_count
+            self._write_state(batch)
+
+        self.logger.info(f'backing up removed {nremoves:,d} history entries')
+
+    def get_txnums(self, hashX: bytes, limit: int = 1000) -> Iterable[int]:
+        '''Generator that returns an unpruned, sorted list of tx_nums in the
+        history of a hashX.  Includes both spending and receiving
+        transactions.  By default yields at most 1000 entries.  Set
+        limit to None to get them all.  '''
+        limit = util.resolve_limit(limit)
+        prefix = b'H' + hashX
+        for db_key, db_val in self.db.iterator(prefix=prefix):
+            tx_numb = db_key[-TXNUM_LEN:]
+            if limit == 0:
+                return
+            tx_num = unpack_txnum(tx_numb)
+            yield tx_num
+            limit -= 1

src/electrumx/server/mempool.py

@@ -0,0 +1,626 @@
+# Copyright (c) 2016-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Mempool handling.'''
+
+import itertools
+import time
+from abc import ABC, abstractmethod
+import asyncio
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set, Iterable
+import math
+from graphlib import TopologicalSorter
+
+from aiorpcx import run_in_thread, sleep, ignore_after
+
+from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash
+from electrumx.lib.tx import SkipTxDeserialize
+from electrumx.lib.util import class_logger, chunks, OldTaskGroup, LogTimeTaken
+from electrumx.lib.tx import TXOSpendStatus, TxOutpoint
+from electrumx.server.db import UTXO
+
+if TYPE_CHECKING:
+    from electrumx.lib.coins import Coin
+
+
+DB_UTXO_MAP = dict[TxOutpoint, Optional[tuple[bytes, int]]]  # prevout->(hashX,value_in_sats)
+
+
+@dataclass(slots=True)
+class MemPoolTx:
+    prevouts: Sequence[TxOutpoint]  # (txid_rev, txout_idx)
+    # A pair is a (hashX, value) tuple
+    in_pairs: Optional[Sequence[tuple[bytes, int]]]  # (hashX, value_in_sats)
+    out_pairs: Sequence[tuple[bytes, int]]  # (hashX, value_in_sats)
+    fee: int  # in sats
+    size: int  # in vbytes
+
+
+@dataclass(slots=True)
+class MemPoolTxSummary:
+    txid_rev: bytes
+    fee: int  # in sats
+    has_unconfirmed_inputs: bool
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class RecentMemPoolTx:
+    txid_rev: bytes
+    fee: int         # in sats
+    vsize: int       # in vbytes
+
+
+class DBSyncError(Exception):
+    pass
+
+
+class MemPoolAPI(ABC):
+    '''A concrete instance of this class is passed to the MemPool object
+    and used by it to query DB and blockchain state.'''
+
+    @abstractmethod
+    async def height(self) -> int:
+        '''Query bitcoind for its height.'''
+
+    @abstractmethod
+    def cached_height(self) -> Optional[int]:
+        '''Return the height of bitcoind the last time it was queried,
+        for any reason, without actually querying it.
+        '''
+
+    @abstractmethod
+    async def daemon_height_changed(self) -> None:
+        '''Wait until bitcoind's height changes.'''
+
+    @abstractmethod
+    def db_height(self) -> int:
+        '''Return the height flushed to the on-disk DB.'''
+
+    @abstractmethod
+    async def db_height_changed(self) -> None:
+        '''Wait until the on-disk DB height changes.'''
+
+    @abstractmethod
+    async def mempool_txids_hum(self) -> Sequence[str]:
+        '''Query bitcoind for the txids of all transactions in its
+        mempool, returned as a list.'''
+
+    @abstractmethod
+    async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]:
+        '''Query bitcoind for the serialized raw transactions with the given
+        txids.  Missing transactions are returned as None.
+
+        txids_hum is an iterable of hexadecimal hash strings.'''
+
+    @abstractmethod
+    async def lookup_utxos(self, prevouts: Sequence[TxOutpoint]) -> Sequence[Optional[Tuple[bytes, int]]]:
+        '''Return a list of (hashX, value) pairs, one for each prevout if unspent,
+        otherwise return None if spent or not found (for the given prevout).
+
+        prevouts - an iterable of (txid_rev, txout_idx) pairs
+        '''
+
+    @abstractmethod
+    async def on_mempool(
+            self,
+            *,
+            touched_hashxs: Set[bytes],
+            touched_outpoints: Set[TxOutpoint],
+            height: int,
+    ):
+        '''Called each time the mempool is synchronized.  touched_hashxs and
+        touched_outpoints are sets of hashXs and tx outpoints touched since
+        the previous call.  height is the  daemon's height at the time the
+        mempool was obtained.
+        '''
+
+
+class MemPool:
+    '''Representation of the daemon's mempool.
+
+        coin - a coin class from coins.py
+        api - an object implementing MemPoolAPI
+
+    Updated regularly in caught-up state.  Goal is to enable efficient
+    response to the calls in the external interface.  To that end we
+    maintain the following maps:
+
+       tx:     txid_rev -> MemPoolTx
+       hashXs: hashX   -> set of all txids_rev of txs touching the hashX
+    '''
+
+    def __init__(
+            self,
+            coin: Type['Coin'],
+            api: MemPoolAPI,
+            *,
+            refresh_secs=5.0,
+            log_status_secs=60.0,
+    ):
+        assert isinstance(api, MemPoolAPI)
+        self.coin = coin
+        self.api = api
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.txs = {}  # type: Dict[bytes, MemPoolTx]  # txid_rev->tx
+        self.hashXs = defaultdict(set)  # type: Dict[Optional[bytes], Set[bytes]]  # hashX->txids_rev
+        self.txo_to_spender = {}  # type: Dict[TxOutpoint, bytes]  # prevout->txid_rev
+        self.cached_compact_histogram = []  # type: Sequence[tuple[float, int]]
+        self.refresh_secs = refresh_secs
+        self.log_status_secs = log_status_secs
+        # Prevents mempool refreshes during fee histogram calculation
+        self.lock = asyncio.Lock()
+
+    async def _logging(self, synchronized_event):
+        '''Print regular logs of mempool stats.'''
+        self.logger.info('beginning processing of daemon mempool.  '
+                         'This can take some time...')
+        start = time.monotonic()
+        await synchronized_event.wait()
+        elapsed = time.monotonic() - start
+        self.logger.info(f'synced in {elapsed:.2f}s')
+        while True:
+            mempool_size = sum(tx.size for tx in self.txs.values()) / 1_000_000
+            self.logger.info(f'{len(self.txs):,d} txs {mempool_size:.2f} MB, '
+                             f'touching {len(self.hashXs):,d} addresses. '
+                             f'{len(self.txo_to_spender):,d} spends.')
+            await sleep(self.log_status_secs)
+            await synchronized_event.wait()
+
+    async def _refresh_histogram(self, synchronized_event):
+        while True:
+            await synchronized_event.wait()
+            async with self.lock:
+                # Threaded as can be expensive
+                bin_size = self.coin.MEMPOOL_COMPACT_HISTOGRAM_BINSIZE
+                await run_in_thread(self._update_histogram, bin_size)
+            await sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
+
+    def _update_histogram(self, bin_size):
+        # Build a histogram by fee rate
+        t0 = time.monotonic()
+        histogram = defaultdict(int)
+        for tx in self.txs.values():
+            fee_rate = tx.fee / tx.size
+            # use 0.1 sat/byte resolution
+            # note: rounding *down* is intentional. This ensures txs
+            #       with a given fee rate will end up counted in the expected
+            #       bucket/interval of the compact histogram.
+            fee_rate = math.floor(10 * fee_rate) / 10
+            histogram[fee_rate] += tx.size
+
+        compact = self._compress_histogram(histogram, bin_size=bin_size)
+        self.logger.info(f'compact fee histogram: {compact}. took {time.monotonic() - t0:.3f}s.')
+        self.cached_compact_histogram = compact
+
+    @classmethod
+    def _compress_histogram(
+            cls, histogram: Dict[float, int], *, bin_size: int
+    ) -> Sequence[Tuple[float, int]]:
+        '''Calculate and return a compact fee histogram as needed for
+        "mempool.get_fee_histogram" protocol request.
+
+        histogram: feerate (sat/vbyte) -> total size in bytes of txs that pay approx feerate
+        bin_size: ~minimum vsize of a bucket of txs in the result (e.g. 100 kb)
+        '''
+        # Now compact it.  For efficiency, get_fees returns a
+        # compact histogram with variable bin size.  The compact
+        # histogram is an array of (fee_rate, vsize) values.
+        # vsize_n is the cumulative virtual size of mempool
+        # transactions with a fee rate in the interval
+        # [rate_(n-1), rate_n)], and rate_(n-1) > rate_n.
+        # Intervals are chosen to create tranches containing at
+        # least a certain cumulative size (bin_size) of transactions.
+        assert bin_size > 0
+        compact = []
+        cum_size = 0
+        prev_fee_rate = None
+        for fee_rate, size in sorted(histogram.items(), reverse=True):
+            # if there is a big lump of txns at this specific size,
+            # consider adding the previous item now (if not added already)
+            if size > 2 * bin_size and prev_fee_rate is not None and cum_size > 0:
+                compact.append((prev_fee_rate, cum_size))
+                cum_size = 0
+                bin_size *= 1.1
+            # now consider adding this item
+            cum_size += size
+            if cum_size > bin_size:
+                compact.append((fee_rate, cum_size))
+                cum_size = 0
+                bin_size *= 1.1
+            prev_fee_rate = fee_rate
+        return compact
+
+    def _accept_transactions(
+            self,
+            *,
+            tx_map: Dict[bytes, MemPoolTx],  # txid_rev->tx
+            utxo_map: DB_UTXO_MAP,  # prevout->(hashX,value_in_sats)
+            touched_hashxs: Set[bytes],  # set of hashXs
+            touched_outpoints: Set[TxOutpoint],  # set of outpoints
+            topologically_sort: bool,
+    ) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]:
+        '''Accept transactions in tx_map to the mempool if all their inputs
+        can be found in the existing mempool or a utxo_map from the
+        DB.
+
+        Returns an (unprocessed tx_map, unspent utxo_map) pair.
+        '''
+        hashXs = self.hashXs
+        txs = self.txs
+        txo_to_spender = self.txo_to_spender
+
+        if topologically_sort:
+            # Sort tx_map so that parent txs come first (in case of unconf chain).
+            # This is just an optimization so that fewer txs will get "deferred", leading to better perf.
+            # Dependencies outside tx_map (already in chainstate (utxo_map), or our mempool (self.txs),
+            # or MISSING) are ignored.
+            cand_to_parents = {
+                txid: {parent_txid for (parent_txid, txout_idx) in tx.prevouts
+                       if parent_txid in tx_map}
+                for (txid, tx) in tx_map.items()}
+            cand_txids = list(TopologicalSorter(cand_to_parents).static_order())
+            assert len(tx_map) == len(cand_to_parents) == len(cand_txids), \
+                f"{len(tx_map)=}, {len(cand_to_parents)=}, {len(cand_txids)=}"
+            del cand_to_parents
+        else:
+            cand_txids = tx_map.keys()
+
+        deferred = {}  # type: dict[bytes, MemPoolTx]
+        unspent = set(utxo_map)
+        # Try to find all prevouts so we can accept the candidate TXs from tx_map into our mempool
+        for txid_rev in cand_txids:
+            tx = tx_map[txid_rev]
+            in_pairs = []
+            try:
+                for prevout in tx.prevouts:
+                    # first, look for parent tx among confirmed UTXOs:
+                    utxo = utxo_map.get(prevout)
+                    if not utxo:  # second, look for parent tx in mempool
+                        prev_hash, prev_index = prevout
+                        # Raises KeyError if prev_hash is not in txs
+                        utxo = txs[prev_hash].out_pairs[prev_index]
+                    in_pairs.append(utxo)
+            except KeyError:
+                deferred[txid_rev] = tx
+                continue
+
+            # Spend the prevouts
+            unspent.difference_update(tx.prevouts)
+
+            # Save the in_pairs, compute the fee and accept the TX
+            tx.in_pairs = tuple(in_pairs)
+            # Avoid negative fees if dealing with generation-like transactions
+            # because some in_parts would be missing
+            tx.fee = max(0, (sum(v for _, v in tx.in_pairs) -
+                             sum(v for _, v in tx.out_pairs)))
+            txs[txid_rev] = tx
+
+            for hashX, _value in itertools.chain(tx.in_pairs, tx.out_pairs):
+                touched_hashxs.add(hashX)
+                hashXs[hashX].add(txid_rev)
+            for prevout in tx.prevouts:
+                txo_to_spender[prevout] = txid_rev
+                touched_outpoints.add(prevout)
+            for out_idx, out_pair in enumerate(tx.out_pairs):
+                touched_outpoints.add((txid_rev, out_idx))
+
+        return deferred, {prevout: utxo_map[prevout] for prevout in unspent}
+
+    async def _refresh_hashes(self, synchronized_event):
+        '''Refresh our view of the daemon's mempool.'''
+        # touched_* accumulates between calls to on_mempool and each
+        # call transfers ownership
+        touched_hashxs = set()
+        touched_outpoints = set()
+        prev_mempool_height = -1
+        while True:
+            height = self.api.cached_height()
+            is_new_block = height != prev_mempool_height
+            with LogTimeTaken(self.logger, "getrawmempool RPC", enabled=is_new_block):
+                txids_hum = await self.api.mempool_txids_hum()
+            if height != await self.api.height():  # if height changed *again*, re-start
+                continue
+            with LogTimeTaken(self.logger, "convert_txids", enabled=is_new_block):
+                txids_rev = await run_in_thread(lambda: {hex_str_to_hash(hh) for hh in txids_hum})
+            if self.api.db_height() < height:
+                # DB not yet caught up. give it some time...
+                async with ignore_after(self.refresh_secs):
+                    await self.api.db_height_changed()
+            try:
+                async with self.lock:
+                    await self._process_mempool(
+                        all_txids_rev=txids_rev,
+                        touched_hashxs=touched_hashxs,
+                        touched_outpoints=touched_outpoints,
+                        mempool_height=height,
+                        debug_logs=is_new_block,
+                    )
+            except DBSyncError:
+                # The UTXO DB is not at the same height as the
+                # mempool; wait and try again
+                self.logger.debug('waiting for DB to sync')
+            else:
+                synchronized_event.set()
+                synchronized_event.clear()
+                await self.api.on_mempool(
+                    touched_hashxs=touched_hashxs,
+                    touched_outpoints=touched_outpoints,
+                    height=height,
+                )
+                prev_mempool_height = height
+                touched_hashxs = set()
+                touched_outpoints = set()
+            # poll bitcoind's whole mempool every few seconds; or instantly after the DB height changes
+            async with ignore_after(self.refresh_secs):
+                async with OldTaskGroup(wait=any) as group:
+                    await group.spawn(self.api.db_height_changed())
+                    await group.spawn(self.api.daemon_height_changed())
+
+    async def _process_mempool(
+            self,
+            *,
+            all_txids_rev: Set[bytes],  # set of txids_rev  # complete view of daemon's mempool
+            touched_hashxs: Set[bytes],  # set of hashXs
+            touched_outpoints: Set[TxOutpoint],  # set of outpoints
+            mempool_height: int,
+            debug_logs: bool = False,
+    ) -> None:
+        # Re-sync with the new set of hashes
+        txs = self.txs
+        hashXs = self.hashXs
+        txo_to_spender = self.txo_to_spender
+
+        if mempool_height != self.api.db_height():  # FIXME should compare blockhash
+            raise DBSyncError
+
+        # 1. Handle txs that have disappeared (evicted, just got mined, etc)
+        # TODO split disappeared txs workload into a threadpool, chunks of ~200 txs
+        def handle_disappeared_txs() -> None:
+            nonlocal touched_hashxs
+            for txid_rev in disappeared_hashes:
+                tx = txs.pop(txid_rev)
+                # hashXs
+                tx_hashXs = {hashX for hashX, value in tx.in_pairs}
+                tx_hashXs.update(hashX for hashX, value in tx.out_pairs)
+                for hashX in tx_hashXs:
+                    hashXs[hashX].remove(txid_rev)
+                    if not hashXs[hashX]:
+                        del hashXs[hashX]
+                touched_hashxs |= tx_hashXs
+                # outpoints
+                for prevout in tx.prevouts:
+                    del txo_to_spender[prevout]
+                    touched_outpoints.add(prevout)
+                for out_idx, out_pair in enumerate(tx.out_pairs):
+                    touched_outpoints.add((txid_rev, out_idx))
+
+        with LogTimeTaken(self.logger, "_process_mempool() removing txs", enabled=debug_logs):
+            disappeared_hashes = await run_in_thread(lambda: set(txs) - all_txids_rev)
+            handle_disappeared_txs()
+
+        # 2. Process new transactions
+        t0 = time.monotonic()
+        new_hashes = await run_in_thread(lambda: list(all_txids_rev.difference(txs)))
+        if new_hashes:
+            # 2.1. fetch raw txs from bitcoin daemon
+            group = OldTaskGroup()
+            for hashes in chunks(new_hashes, 200):
+                coro = self._fetch_raw_txs_and_utxos(
+                    new_txids_rev=hashes,
+                    all_txids_rev=all_txids_rev,
+                )
+                await group.spawn(coro)
+            if mempool_height != self.api.db_height():
+                raise DBSyncError
+
+            tx_map = {}  # type: dict[bytes, MemPoolTx]
+            utxo_map = {}  # type: DB_UTXO_MAP
+            async for task in group:
+                partial_tx_map, partial_utxo_map = task.result()
+                tx_map.update(partial_tx_map)
+                utxo_map.update(partial_utxo_map)
+            if debug_logs:
+                self.logger.debug(f'_process_mempool() fetching txs took {time.monotonic() - t0:.3f}s. {len(new_hashes)=}')
+
+            # 2.2. accept txs into our mempool
+            def accept_txs_loop() -> None:
+                # Accept candidate txs from tx_map into our mempool.
+                # We only accept candidates for which we can find a UTXO for each tx input:
+                # - some UTXOs we find in the DB=utxo_map (chainstate),
+                # - some we find in self.txs (our memool),
+                # - some we might only find in tx_map (among the candidates: consider long chain of unconfirmed txs)
+                # - some we might not find anywhere: if DB is corrupted or bitcoind gave inconsistent mempool, or other race
+                # In each iteration, we loop over tx_map and move accepted transactions from it to self.txs.
+                # In the worst degenerate case, this could be O(n^2). To avoid that, we topologically sort tx_map.
+                nonlocal tx_map, utxo_map
+                prior_txmap_size = 0
+                while tx_map and len(tx_map) != prior_txmap_size:
+                    prior_txmap_size = len(tx_map)
+                    tx_map, utxo_map = self._accept_transactions(
+                        tx_map=tx_map,
+                        utxo_map=utxo_map,
+                        touched_hashxs=touched_hashxs,
+                        touched_outpoints=touched_outpoints,
+                        topologically_sort=True,
+                    )
+                if tx_map:
+                    self.logger.error(f'{len(tx_map)} txs dropped')
+
+            with LogTimeTaken(self.logger, f"_process_mempool() accepting txs ({len(new_hashes)=})", enabled=debug_logs):
+                accept_txs_loop()
+
+    async def _fetch_raw_txs_and_utxos(
+            self,
+            *,
+            new_txids_rev: set[bytes],  # (some) new candidate txs for our mempool
+            all_txids_rev: set[bytes],  # complete view of daemon's mempool
+    ) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]:
+        '''Fetch a list of mempool transactions, and lookup corresponding UTXOs in the DB.'''
+        txids_hum_iter = (hash_to_hex_str(hash) for hash in new_txids_rev)
+        raw_txs = await self.api.raw_transactions(txids_hum_iter)
+
+        def deserialize_txs() -> Dict[bytes, MemPoolTx]:
+            """This function is pure"""
+            to_hashX = self.coin.hashX_from_script
+            deserializer = self.coin.DESERIALIZER
+
+            txs = {}  # type: Dict[bytes, MemPoolTx]
+            for txid_rev, raw_tx in zip(new_txids_rev, raw_txs):
+                # The daemon may have evicted the tx from its
+                # mempool or it may have gotten in a block
+                if not raw_tx:
+                    continue
+                try:
+                    tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
+                except SkipTxDeserialize as ex:
+                    self.logger.debug(f'skipping tx {hash_to_hex_str(txid_rev)}: {ex}')
+                    continue
+                # Convert the inputs and outputs into (hashX, value) pairs
+                # Drop generation-like inputs from MemPoolTx.prevouts
+                txin_pairs = tuple((txin.prev_txid_rev, txin.prev_idx)
+                                   for txin in tx.inputs
+                                   if not txin.is_generation())
+                txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
+                                    for txout in tx.outputs)
+                txs[txid_rev] = MemPoolTx(
+                    prevouts=txin_pairs,
+                    in_pairs=None,
+                    out_pairs=txout_pairs,
+                    fee=0,
+                    size=tx_size,
+                )
+            return txs
+
+        # Thread this potentially slow operation so as not to block
+        tx_map = await run_in_thread(deserialize_txs)  # type: Dict[bytes, MemPoolTx]
+
+        # Determine all prevouts not in the mempool, and fetch the
+        # UTXO information from the database.  Failed prevout lookups
+        # return None - concurrent database updates happen - which is
+        # relied upon by _accept_transactions. Ignore prevouts that are
+        # generation-like.
+        prevouts = tuple(prevout for tx in tx_map.values()
+                         for prevout in tx.prevouts
+                         if prevout[0] not in all_txids_rev)
+        utxos = await self.api.lookup_utxos(prevouts)
+        utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)}
+
+        return tx_map, utxo_map
+
+    #
+    # External interface
+    #
+
+    async def keep_synchronized(self, synchronized_event: asyncio.Event) -> None:
+        '''Keep the mempool synchronized with the daemon.'''
+        async with OldTaskGroup() as group:
+            await group.spawn(self._refresh_hashes(synchronized_event))
+            await group.spawn(self._refresh_histogram(synchronized_event))
+            await group.spawn(self._logging(synchronized_event))
+
+    async def balance_delta(self, hashX: bytes) -> int:
+        '''Return the unconfirmed amount in the mempool for hashX.
+
+        Can be positive or negative.
+        '''
+        value = 0
+        if hashX in self.hashXs:
+            for txid_rev in self.hashXs[hashX]:
+                tx = self.txs[txid_rev]
+                value -= sum(v for h168, v in tx.in_pairs if h168 == hashX)
+                value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
+        return value
+
+    async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]:
+        '''Return a compact fee histogram of the current mempool.'''
+        return self.cached_compact_histogram
+
+    async def potential_spends(self, hashX: bytes) -> set[TxOutpoint]:
+        '''Return a set of (prev_hash, prev_idx) pairs from mempool
+        transactions that touch hashX.
+
+        None, some or all of these may be spends of the hashX, but all
+        actual spends of it (in the DB or mempool) will be included.
+        '''
+        result = set()
+        for txid_rev in self.hashXs.get(hashX, ()):
+            tx = self.txs[txid_rev]
+            result.update(tx.prevouts)
+        return result
+
+    async def transaction_summaries(self, hashX: bytes) -> Sequence[MemPoolTxSummary]:
+        '''Return a list of MemPoolTxSummary objects for the hashX,
+        sorted as expected by protocol methods.
+        '''
+        result = []  # type: list[MemPoolTxSummary]
+        for txid_rev in self.hashXs.get(hashX, ()):
+            tx = self.txs[txid_rev]
+            has_ui = any(hash in self.txs for hash, idx in tx.prevouts)
+            result.append(MemPoolTxSummary(txid_rev, tx.fee, has_ui))
+        result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.txid_rev[::-1]))
+        return result
+
+    async def unordered_UTXOs(self, hashX: bytes) -> Sequence[UTXO]:
+        '''Return an unordered list of UTXO named tuples from mempool
+        transactions that pay to hashX.
+
+        This does not consider if any other mempool transactions spend
+        the outputs.
+        '''
+        utxos = []
+        for txid_rev in self.hashXs.get(hashX, ()):
+            tx = self.txs.get(txid_rev)
+            for pos, (hX, value) in enumerate(tx.out_pairs):
+                if hX == hashX:
+                    utxos.append(UTXO(-1, pos, txid_rev, 0, value))
+        return utxos
+
+    async def spender_for_txo(self, prev_txid_rev: bytes, txout_idx: int) -> 'TXOSpendStatus':
+        '''For an outpoint, returns its spend-status.
+        This only considers the mempool, not the DB/blockchain, so e.g. mined
+        txs are not distinguished from txs that never existed.
+        '''
+        # look up funding tx
+        prev_tx = self.txs.get(prev_txid_rev, None)
+        if prev_tx is None:
+            # funding tx already mined or never existed
+            funder_height = None
+        else:
+            if len(prev_tx.out_pairs) <= txout_idx:
+                # output idx out of bounds...?
+                return TXOSpendStatus(funder_height=None)
+            prev_has_ui = any(hash in self.txs for hash, idx in prev_tx.prevouts)
+            funder_height = -prev_has_ui
+        prevout = (prev_txid_rev, txout_idx)
+        # look up spending tx
+        spender_txid_rev = self.txo_to_spender.get(prevout, None)
+        if spender_txid_rev is None:
+            return TXOSpendStatus(funder_height=funder_height)
+        spender_tx = self.txs.get(spender_txid_rev, None)
+        if spender_tx is None:
+            self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txid_rev)} not in"
+                                f"mempool, but txo_to_spender referenced it as spender "
+                                f"of {hash_to_hex_str(prev_txid_rev)}:{txout_idx} ?!")
+            return TXOSpendStatus(funder_height=funder_height)
+        spender_has_ui = any(txid_rev in self.txs for txid_rev, idx in spender_tx.prevouts)
+        spender_height = -spender_has_ui
+        return TXOSpendStatus(
+            funder_height=funder_height,
+            spender_txid_rev=spender_txid_rev,
+            spender_height=spender_height,
+        )
+
+    async def get_recently_added_txs(self, *, count: int) -> Sequence[RecentMemPoolTx]:
+        # note: inefficient for large "count"s
+        it = reversed(self.txs.items())
+        count = min(count, len(self.txs))
+        mempool_txs = [next(it) for _ in range(count)]
+        return [
+            RecentMemPoolTx(txid_rev=hash, fee=mtx.fee, vsize=mtx.size)
+            for hash, mtx in mempool_txs]

src/electrumx/server/peers.py

@@ -0,0 +1,634 @@
+# Copyright (c) 2017-2018, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Peer management.'''
+
+import asyncio
+import random
+import socket
+import ssl
+import time
+from collections import Counter, defaultdict
+from ipaddress import IPv4Address, IPv6Address
+from typing import TYPE_CHECKING, Type, Any, Optional
+from functools import partial
+
+import aiohttp
+from aiorpcx import (Event, Notification, RPCSession, SOCKSError,
+                     SOCKSProxy, TaskTimeout, connect_rs,
+                     handler_invocation, ignore_after, sleep)
+from aiorpcx.jsonrpc import CodeMessageError
+
+from electrumx.lib.peer import Peer
+from electrumx.lib.util import class_logger, json_deserialize, OldTaskGroup
+from electrumx.server.session import RPCSessionWithTaskGroup
+from electrumx.server.transport import PaddedRSTransport
+
+if TYPE_CHECKING:
+    from electrumx.server.env import Env
+    from electrumx.server.db import DB
+
+
+PEER_GOOD, PEER_STALE, PEER_NEVER, PEER_BAD = range(4)
+STATUS_DESCS = ('good', 'stale', 'never', 'bad')
+STALE_SECS = 3 * 3600
+WAKEUP_SECS = 300
+PEER_ADD_PAUSE = 600
+
+
+class BadPeerError(Exception):
+    pass
+
+
+def assert_good(message, result, instance):
+    if not isinstance(result, instance):
+        raise BadPeerError(f'{message} returned bad result type '
+                           f'{type(result).__name__}')
+
+
+class PeerSession(RPCSessionWithTaskGroup):
+    '''An outgoing session to a peer.'''
+
+    async def handle_request(self, request):
+        # We subscribe so might be unlucky enough to get a notification...
+        if (isinstance(request, Notification) and
+                request.method == 'blockchain.headers.subscribe'):
+            pass
+        else:
+            await handler_invocation(None, request)   # Raises
+
+
+class PeerManager:
+    '''Looks after the DB of peer network servers.
+
+    Attempts to maintain a connection with up to 8 peers.
+    Issues a 'peers.subscribe' RPC to them and tells them our data.
+    '''
+    def __init__(self, env: 'Env', db: 'DB'):
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        # Initialise the Peer class
+        Peer.DEFAULT_PORTS = env.coin.PEER_DEFAULT_PORTS
+        self.env = env
+        self.db = db
+
+        # Our reported clearnet and Tor Peers, if any
+        sclass = env.coin.SESSIONCLS
+        self.myselves = [Peer(str(service.host), sclass.server_features(env), 'env')
+                         for service in env.report_services]
+        self.server_version_args = sclass.server_version_args()
+        # Peers have one entry per hostname.  Once connected, the
+        # ip_addr property is either None, an onion peer, or the
+        # IP address that was connected to.  Adding a peer will evict
+        # any other peers with the same host name or IP address.
+        self.peers = set()
+        self.permit_onion_peer_time = time.time()
+        self.proxy = None
+        self.group = OldTaskGroup()
+        self.recent_peer_adds = {}
+        # refreshed
+        self.blacklist = set()
+
+    def _my_clearnet_peer(self):
+        '''Returns the clearnet peer representing this server, if any.'''
+        clearnet = [peer for peer in self.myselves if not peer.is_tor]
+        return clearnet[0] if clearnet else None
+
+    def _set_peer_statuses(self):
+        '''Set peer statuses.'''
+        cutoff = time.time() - STALE_SECS
+        for peer in self.peers:
+            if peer.bad:
+                peer.status = PEER_BAD
+            elif peer.last_good > cutoff:
+                peer.status = PEER_GOOD
+            elif peer.last_good:
+                peer.status = PEER_STALE
+            else:
+                peer.status = PEER_NEVER
+
+    def _features_to_register(self, peer, remote_peers):
+        '''If we should register ourselves to the remote peer, which has
+        reported the given list of known peers, return the clearnet
+        identity features to register, otherwise None.
+        '''
+        # Announce ourself if not present.  Don't if disabled, we
+        # are a non-public IP address, or to ourselves.
+        if not self.env.peer_announce or peer in self.myselves:
+            return None
+        my = self._my_clearnet_peer()
+        if not my or not my.is_public:
+            return None
+        # Register if no matches, or ports have changed
+        for peer in my.matches(remote_peers):
+            if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port:
+                return None
+        return my.features
+
+    def _permit_new_onion_peer(self, now):
+        '''Accept a new onion peer only once per random time interval.'''
+        if now < self.permit_onion_peer_time:
+            return False
+        self.permit_onion_peer_time = now + random.randrange(0, 1200)
+        return True
+
+    async def _import_peers(self):
+        '''Import hard-coded peers from a file or the coin defaults.'''
+        imported_peers = self.myselves.copy()
+        # Add the hard-coded ones unless only reporting ourself
+        if self.env.peer_discovery != self.env.PD_SELF:
+            imported_peers.extend(Peer.from_real_name(real_name, 'coins.py')
+                                  for real_name in self.env.coin.PEERS)
+        await self._note_peers(imported_peers, limit=None)
+
+    async def _refresh_blacklist(self):
+        url = self.env.blacklist_url
+        if not url:
+            return
+
+        async def read_blacklist():
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as response:
+                    text = await response.text()
+            return {entry.lower() for entry in json_deserialize(text)}
+
+        while True:
+            try:
+                self.blacklist = await read_blacklist()
+            except Exception as e:
+                self.logger.error(f'could not retrieve blacklist from {url}: {e}')
+            else:
+                self.logger.info(f'blacklist from {url} has {len(self.blacklist)} entries')
+                # Got new blacklist. Now check our current peers against it
+                for peer in self.peers:
+                    if self._is_blacklisted(peer):
+                        peer.retry_event.set()
+            await sleep(600)
+
+    def _is_blacklisted(self, peer):
+        host = peer.host.lower()
+        second_level_domain = '*.' + '.'.join(host.split('.')[-2:])
+        return any(item in self.blacklist
+                   for item in (host, second_level_domain, peer.ip_addr))
+
+    def _get_recent_good_peers(self):
+        cutoff = time.time() - STALE_SECS
+        recent = [peer for peer in self.peers
+                  if peer.last_good > cutoff and
+                  not peer.bad and peer.is_public]
+        recent = [peer for peer in recent if not self._is_blacklisted(peer)]
+        return recent
+
+    async def _detect_proxy(self):
+        '''Detect a proxy if we don't have one and some time has passed since
+        the last attempt.
+
+        If found self.proxy is set to a SOCKSProxy instance, otherwise None.
+        '''
+        host = self.env.tor_proxy_host
+        if self.env.tor_proxy_port is None:
+            ports = [9050, 9150, 1080]
+        else:
+            ports = [self.env.tor_proxy_port]
+        while True:
+            self.logger.info(f'trying to detect proxy on "{host}" '
+                             f'ports {ports}')
+            proxy = await SOCKSProxy.auto_detect_at_host(host, ports, None)
+            if proxy:
+                self.proxy = proxy
+                self.logger.info(f'detected {proxy}')
+                return
+            self.logger.info('no proxy detected, will try later')
+            await sleep(900)
+
+    async def _note_peers(self, peers, limit=2, check_ports=False, source=None):
+        '''Add a limited number of peers that are not already present.'''
+        new_peers = []
+        match_set = self.peers.copy()
+        for peer in peers:
+            if not peer.is_public or (peer.is_tor and not self.proxy):
+                continue
+
+            matches = peer.matches(match_set)
+            if matches:
+                if check_ports:
+                    for match in matches:
+                        if match.check_ports(peer):
+                            self.logger.info(f'ports changed for {peer}')
+                            # Retry connecting to the peer. First we will try the existing
+                            # ports and then try the new ports. Note that check_ports above
+                            # had a side_effect to temporarily store the new ports.
+                            # If we manage to connect, we will call 'server.features',
+                            # and the ports for this peer will be updated to the return values.
+                            match.retry_event.set()
+            else:
+                match_set.add(peer)
+                new_peers.append(peer)
+
+        if new_peers:
+            source = source or new_peers[0].source
+            if limit:
+                random.shuffle(new_peers)
+                use_peers = new_peers[:limit]
+            else:
+                use_peers = new_peers
+            for peer in use_peers:
+                self.logger.info(f'accepted new peer {peer} from {source}')
+                peer.retry_event = Event()
+                self.peers.add(peer)
+                await self.group.spawn(self._monitor_peer(peer))
+
+        return True
+
+    async def _monitor_peer(self, peer):
+        # Stop monitoring if we were dropped (a duplicate peer)
+        while peer in self.peers:
+            if await self._should_drop_peer(peer):
+                self.peers.discard(peer)
+                break
+            # Figure out how long to sleep before retrying.  Retry a
+            # good connection when it is about to turn stale, otherwise
+            # exponentially back off retries.
+            if peer.try_count == 0:
+                pause = STALE_SECS - WAKEUP_SECS * 2
+            else:
+                pause = WAKEUP_SECS * 2 ** peer.try_count
+            async with ignore_after(pause):
+                await peer.retry_event.wait()
+                peer.retry_event.clear()
+
+    async def _should_drop_peer(self, peer):
+        peer.try_count += 1
+        is_good = False
+        for kind, port, family in peer.connection_tuples():
+            peer.last_try = time.time()
+
+            kwargs = {'family': family}
+            if kind == 'SSL':
+                sslc = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+                sslc.check_hostname = False
+                sslc.verify_mode = ssl.CERT_NONE
+                kwargs['ssl'] = sslc
+
+            if self.env.force_proxy or peer.is_tor:
+                if not self.proxy:
+                    return
+                kwargs['proxy'] = self.proxy
+                kwargs['resolve'] = not peer.is_tor
+            else:
+                # Use our listening Host/IP for outgoing non-proxy
+                # connections so our peers see the correct source.
+                local_hosts = {service.host for service in self.env.services
+                               if isinstance(service.host, (IPv4Address, IPv6Address))
+                               and service.protocol != 'rpc'}
+                if local_hosts:
+                    kwargs['local_addr'] = (str(local_hosts.pop()), None)
+
+            session_factory = partial(
+                PeerSession,
+                manager_taskgroup=self.group,
+            )
+            peer_text = f'[{peer}:{port} {kind}]'
+            try:
+                async with connect_rs(
+                        peer.host, port,
+                        session_factory=session_factory,
+                        transport=PaddedRSTransport,
+                        **kwargs,
+                ) as session:
+                    session.sent_request_timeout = 120 if peer.is_tor else 30
+                    await self._verify_peer(session, peer)
+                is_good = True
+                break
+            except BadPeerError as e:
+                self.logger.error(f'{peer_text} marking bad: ({e})')
+                peer.mark_bad()
+                break
+            except CodeMessageError as e:
+                self.logger.error(f'{peer_text} RPC error: {e.message} '
+                                  f'({e.code})')
+            except (OSError, SOCKSError, ConnectionError, TaskTimeout) as e:
+                self.logger.info(f'{peer_text} {e}')
+
+        if is_good:
+            # Monotonic time would be better, but last_good and last_try are
+            # exported to admin RPC client.
+            now = time.time()
+            elapsed = now - peer.last_try
+            self.logger.info(f'{peer_text} verified in {elapsed:.1f}s')
+            peer.try_count = 0
+            peer.last_good = now
+            peer.source = 'peer'
+            # At most 2 matches if we're a host name, potentially
+            # several if we're an IP address (several instances
+            # can share a NAT).
+            matches = peer.matches(self.peers)
+            for match in matches:
+                if match.ip_address:
+                    if len(matches) > 1:
+                        self.peers.remove(match)
+                        # Force the peer's monitoring task to exit
+                        match.retry_event.set()
+                elif peer.host in match.features['hosts']:
+                    match.update_features_from_peer(peer)
+            # Trim this data structure
+            self.recent_peer_adds = {k: v for k, v in self.recent_peer_adds.items()
+                                     if v + PEER_ADD_PAUSE < now}
+        else:
+            # Forget the peer if long-term unreachable
+            if peer.last_good and not peer.bad:
+                try_limit = 10
+            else:
+                try_limit = 3
+            if peer.try_count >= try_limit:
+                desc = 'bad' if peer.bad else 'unreachable'
+                self.logger.info(f'forgetting {desc} peer: {peer}')
+                return True
+        return False
+
+    async def _verify_peer(self, session, peer):
+        # store IP address for peer
+        if not peer.is_tor:
+            address = session.remote_address()
+            if address and isinstance(address.host, (IPv4Address, IPv6Address)):
+                peer.ip_addr = str(address.host)
+
+        if self._is_blacklisted(peer):
+            raise BadPeerError('blacklisted')
+
+        # Bucket good recent peers; forbid many servers from similar IPs
+        # FIXME there's a race here, when verifying multiple peers
+        #       that belong to the same bucket ~simultaneously
+        recent_peers = self._get_recent_good_peers()
+        if peer in recent_peers:
+            recent_peers.remove(peer)
+        onion_peers = []
+        buckets = defaultdict(list)
+        for other_peer in recent_peers:
+            if other_peer.is_tor:
+                onion_peers.append(other_peer)
+            else:
+                buckets[other_peer.bucket_for_internal_purposes()].append(other_peer)
+        if peer.is_tor:
+            # keep number of onion peers below half of all peers,
+            # but up to 100 is OK regardless
+            if len(onion_peers) > len(recent_peers) // 2 >= 100:
+                raise BadPeerError('too many onion peers already')
+        else:
+            bucket = peer.bucket_for_internal_purposes()
+            if buckets[bucket]:
+                raise BadPeerError(f'too many peers already in bucket {bucket}')
+
+        # server.version goes first
+        message = 'server.version'
+        try:
+            result = await session.send_request(message, self.server_version_args)
+        except asyncio.CancelledError:
+            raise BadPeerError('terminated before server.version response')
+
+        assert_good(message, result, list)
+
+        # Protocol version 1.1 returns a pair with the version first
+        if len(result) != 2 or not all(isinstance(x, str) for x in result):
+            raise BadPeerError(f'bad server.version result: {result}')
+        server_version, _protocol_version = result
+        peer.server_version = server_version
+        peer.features['server_version'] = server_version
+
+        async with OldTaskGroup() as g:
+            await g.spawn(self._send_headers_subscribe(session))
+            await g.spawn(self._send_server_features(session, peer))
+            peers_task = await g.spawn(self._send_peers_subscribe
+                                       (session, peer))
+
+        # Process reported peers if remote peer is good
+        peers = peers_task.result()
+        await self._note_peers(peers)
+
+        features = self._features_to_register(peer, peers)
+        if features:
+            self.logger.info(f'registering ourself with {peer}')
+            # We only care to wait for the response
+            try:
+                await session.send_request('server.add_peer', [features])
+            except asyncio.CancelledError:
+                raise BadPeerError('terminated before server.add_peer response')
+
+    async def _send_headers_subscribe(self, session):
+        message = 'blockchain.headers.subscribe'
+        result = await session.send_request(message)
+        assert_good(message, result, dict)
+
+        # Compare heights.
+        MAX_TIP_HEIGHT_DIFF = 5
+        our_height = self.db.db_height
+        their_height = result.get('height')
+        if not isinstance(their_height, int):
+            raise BadPeerError(f'invalid height {their_height}')
+        if abs(our_height - their_height) > MAX_TIP_HEIGHT_DIFF:
+            raise BadPeerError(f'bad height {their_height:,d} '
+                               f'(ours: {our_height:,d})')
+
+        # Check prior header too in case of hard fork.
+        # Allow short chain-splits. On Bitcoin testnet4, <6 block splits and reorgs are *very* common.
+        MAX_CHAINSPLIT_LEN = 10
+        check_height = max(0, min(our_height, their_height) - MAX_CHAINSPLIT_LEN)
+        raw_header = await self.db.raw_header(check_height)
+        ours = raw_header.hex()
+        message = 'blockchain.block.header'
+        theirs = await session.send_request(message, [check_height])
+        assert_good(message, theirs, str)
+        if ours != theirs:
+            raise BadPeerError(f'our header {ours} and '
+                               f'theirs {theirs} differ')
+
+    async def _send_server_features(self, session, peer):
+        message = 'server.features'
+        features = await session.send_request(message)
+        assert_good(message, features, dict)
+        features_hosts = features.get('hosts', {})
+        assert_good(message, features_hosts, dict)
+        hosts = [host.lower() for host in features_hosts]
+        if self.env.coin.GENESIS_HASH != features.get('genesis_hash'):
+            raise BadPeerError('incorrect genesis hash')
+        if peer.host.lower() in hosts:
+            peer.update_features(features)
+        else:
+            raise BadPeerError(f'not listed in own hosts list {hosts}')
+
+    async def _send_peers_subscribe(self, session, peer):
+        message = 'server.peers.subscribe'
+        raw_peers = await session.send_request(message)
+        assert_good(message, raw_peers, list)
+
+        # Check the peers list we got from a remote peer.
+        # Each is expected to be of the form:
+        #    [ip_addr, hostname, ['v1.0', 't51001', 's51002']]
+        # Call add_peer if the remote doesn't appear to know about us.
+        try:
+            real_names = [' '.join([u[1]] + u[2]) for u in raw_peers]
+            return [Peer.from_real_name(real_name, str(peer))
+                    for real_name in real_names]
+        except Exception:
+            raise BadPeerError('bad server.peers.subscribe response')
+
+    #
+    # External interface
+    #
+    async def discover_peers(self):
+        '''Perform peer maintenance.  This includes
+
+          1) Forgetting unreachable peers.
+          2) Verifying connectivity of new peers.
+          3) Retrying old peers at regular intervals.
+        '''
+        self.logger.info(f'peer discovery: {self.env.peer_discovery}')
+        if self.env.peer_discovery != self.env.PD_ON:
+            self.logger.info('peer discovery is disabled')
+            return
+
+        self.logger.info(f'announce ourself: {self.env.peer_announce}')
+        self.logger.info(f'my clearnet self: {self._my_clearnet_peer()}')
+        self.logger.info(f'force use of proxy: {self.env.force_proxy}')
+        self.logger.info(f'beginning peer discovery...')
+        try:
+            async with self.group as group:
+                await group.spawn(self._refresh_blacklist())
+                await group.spawn(self._detect_proxy())
+                await group.spawn(self._import_peers())
+        except Exception:
+            self.logger.exception(
+                "PeerManager.group died! Please open a bug report about this, "
+                "and include at least this traceback and maybe a bit more of "
+                "the log (if it looks relevant).")
+            while True:
+                await sleep(3600)
+                self.logger.warning(
+                    "Peer discovery is not working as PeerManager.group died. "
+                    "This is not normal. You should inspect the log.")
+
+    def info(self):
+        '''The number of peers.'''
+        self._set_peer_statuses()
+        counter = Counter(peer.status for peer in self.peers)
+        return {
+            'bad': counter[PEER_BAD],
+            'good': counter[PEER_GOOD],
+            'never': counter[PEER_NEVER],
+            'stale': counter[PEER_STALE],
+            'total': len(self.peers),
+        }
+
+    async def add_localRPC_peer(self, real_name):
+        '''Add a peer passed by the admin over LocalRPC.'''
+        await self._note_peers([Peer.from_real_name(real_name, 'RPC')], check_ports=True)
+
+    async def on_add_peer(self, features: dict[str, Any] | Any, source_addr: Optional[str]):
+        '''Add a peer (but only if the peer resolves to the source).'''
+        if self.env.peer_discovery != self.env.PD_ON:
+            return False
+        if not source_addr:
+            self.logger.info('ignored add_peer request: no source info')
+            return False
+        source = str(source_addr.host)
+        peers = Peer.peers_from_features(features, source)
+        if not peers:
+            self.logger.info('ignored add_peer request: no peers given')
+            return False
+
+        # Just look at the first peer, require it
+        peer = peers[0]
+        host = peer.host
+        now = time.time()
+
+        # Rate limit peer adds by domain to one every 10 minutes
+        if peer.ip_address is not None:
+            bucket = 'ip_addr'
+        else:
+            bucket = '.'.join(host.lower().split('.')[-2:])
+        last = self.recent_peer_adds.get(bucket, 0)
+        self.recent_peer_adds[bucket] = now
+        if last + PEER_ADD_PAUSE >= now:
+            return False
+
+        if peer.is_tor:
+            permit = self._permit_new_onion_peer(now)
+            reason = 'rate limiting'
+        else:
+            getaddrinfo = asyncio.get_event_loop().getaddrinfo
+            try:
+                infos = await getaddrinfo(host, 80, type=socket.SOCK_STREAM)
+            except socket.gaierror:
+                permit = False
+                reason = 'address resolution failure'
+            else:
+                permit = any(source == info[-1][0] for info in infos)
+                reason = 'source-destination mismatch'
+
+        if permit:
+            self.logger.info(f'accepted add_peer request from {source} for {host}')
+            await self._note_peers([peer], check_ports=True)
+        else:
+            self.logger.warning(f'rejected add_peer request from {source} '
+                                f'for {host} ({reason})')
+
+        return permit
+
+    def on_peers_subscribe(self, is_tor):
+        '''Returns the server peers as a list of (ip, host, details) tuples.
+
+        We return all peers we've connected to in the last day.
+        Additionally, if we don't have onion routing, we return a few
+        hard-coded onion servers.
+        '''
+        recent = self._get_recent_good_peers()
+
+        # Always report ourselves if valid (even if not public)
+        cutoff = time.time() - STALE_SECS
+        peers = {
+            myself
+            for myself in self.myselves
+            if myself.last_good > cutoff
+        }
+
+        # Bucket the clearnet peers and select up to two from each
+        onion_peers = []
+        buckets = defaultdict(list)
+        for peer in recent:
+            if peer.is_tor:
+                onion_peers.append(peer)
+            else:
+                buckets[peer.bucket_for_external_interface()].append(peer)
+        for bucket_peers in buckets.values():
+            random.shuffle(bucket_peers)
+            peers.update(bucket_peers[:2])
+
+        # Add up to 20% onion peers (but up to 10 is OK anyway)
+        random.shuffle(onion_peers)
+        max_onion = 50 if is_tor else max(10, len(peers) // 4)
+
+        peers.update(onion_peers[:max_onion])
+
+        return [peer.to_tuple() for peer in peers]
+
+    def proxy_address(self):
+        '''Return the NetAddress of the proxy, if there is a proxy, otherwise
+        None.'''
+        return self.proxy.address if self.proxy else None
+
+    def rpc_data(self):
+        '''Peer data for the peers RPC method.'''
+        self._set_peer_statuses()
+
+        def peer_data(peer):
+            data = peer.serialize()
+            data['status'] = STATUS_DESCS[peer.status]
+            return data
+
+        def peer_key(peer):
+            return peer.bad, -peer.last_good
+
+        return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)]

src/electrumx/server/storage.py

@@ -0,0 +1,219 @@
+# Copyright (c) 2016-2017, the ElectrumX authors
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Backend database abstraction.'''
+
+import os
+from functools import partial
+from typing import Type, Sequence, TYPE_CHECKING
+
+import electrumx.lib.util as util
+
+if TYPE_CHECKING:
+    from rocksdb import DB as rocksdb_db
+    from plyvel import DB as plyvel_db
+
+
+def db_class(name: str) -> Type['Storage']:
+    '''Returns a DB engine class.'''
+    for db_class in util.subclasses(Storage):
+        if db_class.__name__.lower() == name.lower():
+            db_class.import_module()
+            return db_class
+    raise RuntimeError(f'unrecognised DB engine "{name}"')
+
+
+def list_db_engine_choices() -> Sequence[str]:
+    return [db_class.__name__.lower() for db_class in util.subclasses(Storage)]
+
+
+class Storage:
+    '''Abstract base class of the DB backend abstraction.'''
+
+    def __init__(self, name: str, for_sync: bool):
+        self.is_new = not os.path.exists(name)
+        self.for_sync = for_sync or self.is_new
+        self.open(name, create=self.is_new)
+
+    @classmethod
+    def import_module(cls) -> None:
+        '''Import the DB engine module.'''
+        raise NotImplementedError
+
+    def open(self, name: str, create: bool):
+        '''Open an existing database or create a new one.'''
+        raise NotImplementedError
+
+    def close(self) -> None:
+        '''Close an existing database.'''
+        raise NotImplementedError
+
+    def get(self, key: bytes) -> bytes:
+        raise NotImplementedError
+
+    def put(self, key: bytes, value: bytes) -> None:
+        raise NotImplementedError
+
+    def write_batch(self):
+        '''Return a context manager that provides `put` and `delete`.
+
+        Changes should only be committed when the context manager
+        closes without an exception.
+        '''
+        raise NotImplementedError
+
+    def iterator(self, prefix: bytes = b'', reverse: bool = False):
+        '''Return an iterator that yields (key, value) pairs from the
+        database sorted by key.
+
+        If `prefix` is set, only keys starting with `prefix` will be
+        included.  If `reverse` is True the items are returned in
+        reverse order.
+
+        The iterator supports .seek(key), which moves it just left of `key`.
+        - if forward-iterating
+            - if `key` is present, it will be the next item
+            - if `key` is not present, the next item will be the smallest still greater than `key`
+        - if reverse-iterating
+            - the next item will be the largest still smaller than `key`
+        '''
+        raise NotImplementedError
+
+
+class LevelDB(Storage):
+    '''LevelDB database engine.'''
+
+    db: 'plyvel_db'
+
+    @classmethod
+    def import_module(cls):
+        try:
+            import plyvel
+        except ImportError as e:
+            raise Exception(f"{e}. For the leveldb DB-backend, you should install ElectrumX with the [leveldb] pip extra.") from e
+
+        cls.module = plyvel
+
+    def open(self, name, create):
+        mof = 512 if self.for_sync else 128
+        # Use snappy compression (the default)
+        self.db = self.module.DB(name, create_if_missing=create,
+                                 max_open_files=mof)
+        self.close = self.db.close
+        self.get = self.db.get
+        self.put = self.db.put
+        self.write_batch = partial(self.db.write_batch, transaction=True,
+                                   sync=True)
+
+    def iterator(self, prefix=b'', reverse=False):
+        return LevelDBIterator(db=self.db, prefix=prefix, reverse=reverse)
+
+
+class LevelDBIterator:
+    '''An iterator for LevelDB.'''
+
+    def __init__(self, *, db: 'plyvel_db', prefix: bytes, reverse: bool):
+        self.prefix = prefix
+        self.iterator = db.iterator(prefix=prefix, reverse=reverse)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        k, v = next(self.iterator)
+        if not k.startswith(self.prefix):
+            raise StopIteration
+        return k, v
+
+    def seek(self, key: bytes) -> None:
+        self.iterator.seek(key)
+
+
+class RocksDB(Storage):
+    '''RocksDB database engine.'''
+
+    db: 'rocksdb_db'
+
+    @classmethod
+    def import_module(cls):
+        try:
+            import rocksdb
+        except ImportError as e:
+            raise Exception(f"{e}. For the rocksdb DB-backend, you should install ElectrumX with the [rocksdb] pip extra.") from e
+        cls.module = rocksdb
+
+    def open(self, name, create):
+        mof = 512 if self.for_sync else 128
+        # Use snappy compression (the default)
+        options = self.module.Options(create_if_missing=create,
+                                      use_fsync=True,
+                                      target_file_size_base=33554432,
+                                      max_open_files=mof)
+        self.db = self.module.DB(name, options)
+        self.close = self.db.close
+        self.get = self.db.get
+        self.put = self.db.put
+
+    def write_batch(self):
+        return RocksDBWriteBatch(self.db)
+
+    def iterator(self, prefix=b'', reverse=False):
+        return RocksDBIterator(db=self.db, prefix=prefix, reverse=reverse)
+
+
+class RocksDBWriteBatch:
+    '''A write batch for RocksDB.'''
+
+    def __init__(self, db: 'rocksdb_db'):
+        self.batch = RocksDB.module.WriteBatch()
+        self.db = db
+
+    def __enter__(self):
+        return self.batch
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not exc_val:
+            self.db.write(self.batch)
+
+
+class RocksDBIterator:
+    '''An iterator for RocksDB.'''
+
+    def __init__(self, *, db: 'rocksdb_db', prefix: bytes, reverse: bool):
+        self.prefix = prefix
+        self._is_reverse = reverse
+        if reverse:
+            self.iterator = reversed(db.iteritems())
+            nxt_prefix = util.increment_byte_string(prefix)
+            if nxt_prefix:
+                self.iterator.seek(nxt_prefix)
+                try:
+                    next(self.iterator)
+                except StopIteration:
+                    self.iterator.seek(nxt_prefix)
+            else:
+                self.iterator.seek_to_last()
+        else:
+            self.iterator = db.iteritems()
+            self.iterator.seek(prefix)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        k, v = next(self.iterator)
+        if not k.startswith(self.prefix):
+            raise StopIteration
+        return k, v
+
+    def seek(self, key: bytes) -> None:
+        self.iterator.seek(key)
+        if self._is_reverse:
+            try:
+                next(self)
+            except StopIteration:
+                self.iterator.seek_to_last()

src/electrumx/server/transport.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2025, The Electrum developers
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+import asyncio
+import time
+from typing import Optional, TYPE_CHECKING
+
+from aiorpcx.rawsocket import RSTransport
+
+if TYPE_CHECKING:
+    from .session import RPCSessionWithTaskGroup
+
+
+class PaddedRSTransport(RSTransport):
+    """A raw socket transport that provides basic countermeasures against traffic analysis
+    by padding the jsonrpc payload with whitespaces to have ~uniform-size TCP packets.
+    (it is assumed that a network observer does not see plaintext transport contents,
+    due to it being wrapped e.g. in TLS)
+    """
+
+    MIN_PACKET_SIZE = 1024
+    WAIT_FOR_BUFFER_GROWTH_SECONDS = 1.0
+    # amount of (unpadded) bytes sent instantly before beginning with polling.
+    # This makes the initial handshake where a few small messages are exchanged faster.
+    WARMUP_BUDGET_SIZE = 1024
+
+    session: Optional['RPCSessionWithTaskGroup']
+
+    def __init__(self, *args, **kwargs):
+        RSTransport.__init__(self, *args, **kwargs)
+        self._sbuffer = bytearray()  # "send buffer"
+        self._sbuffer_task = None  # type: Optional[asyncio.Task]
+        self._sbuffer_has_data_evt = asyncio.Event()
+        self._last_send = time.monotonic()
+        self._force_send = False  # type: bool
+
+    # note: this does not call super().write() but is a complete reimplementation
+    async def write(self, message):
+        await self._can_send.wait()
+        if self.is_closing():
+            return
+        framed_message = self._framer.frame(message)
+        self._sbuffer += framed_message
+        self._sbuffer_has_data_evt.set()
+        self._maybe_consume_sbuffer()
+
+    def _maybe_consume_sbuffer(self) -> None:
+        """Maybe take some data from sbuffer and send it on the wire."""
+        if not self._can_send.is_set() or self.is_closing():
+            return
+        buf = self._sbuffer
+        if not buf:
+            return
+        # if there is enough data in the buffer, or if we haven't sent in a while, send now:
+        if not (
+            self._force_send
+            or len(buf) >= self.MIN_PACKET_SIZE
+            or self._last_send + self.WAIT_FOR_BUFFER_GROWTH_SECONDS < time.monotonic()
+            or self.session.send_size < self.WARMUP_BUDGET_SIZE
+        ):
+            return
+        assert buf[-2:] in (b"}\n", b"]\n"), f"unexpected json-rpc terminator: {buf[-2:]=!r}"
+        # either (1) pad length to next power of two, to create "lsize" packet:
+        payload_lsize = len(buf)
+        total_lsize = max(self.MIN_PACKET_SIZE, 2 ** (payload_lsize.bit_length()))
+        npad_lsize = total_lsize - payload_lsize
+        # or if that wasted a lot of bandwidth with padding, (2) defer sending some messages
+        # and create a packet with half that size ("ssize", s for small)
+        total_ssize = max(self.MIN_PACKET_SIZE, total_lsize // 2)
+        payload_ssize = buf.rfind(b"\n", 0, total_ssize)
+        if payload_ssize != -1:
+            payload_ssize += 1  # for "\n" char
+            npad_ssize = total_ssize - payload_ssize
+        else:
+            npad_ssize = float("inf")
+        # decide between (1) and (2):
+        if self._force_send or npad_lsize <= npad_ssize:
+            # (1) create "lsize" packet: consume full buffer
+            npad = npad_lsize
+            p_idx = payload_lsize
+        else:
+            # (2) create "ssize" packet: consume some, but defer some for later
+            npad = npad_ssize
+            p_idx = payload_ssize
+        # pad by adding spaces near end
+        json_rpc_terminator = buf[p_idx-2:p_idx]
+        assert json_rpc_terminator in (b"}\n", b"]\n"), f"unexpected {json_rpc_terminator=!r}"
+        buf2 = buf[:p_idx-2] + (npad * b" ") + json_rpc_terminator
+        self._asyncio_transport.write(buf2)
+        self._last_send = time.monotonic()
+        del self._sbuffer[:p_idx]
+        if not self._sbuffer:
+            self._sbuffer_has_data_evt.clear()
+
+    async def _poll_sbuffer(self):
+        while not self.is_closing():
+            await self._can_send.wait()
+            await self._sbuffer_has_data_evt.wait()  # to avoid busy-waiting
+            self._maybe_consume_sbuffer()
+            # If there is still data in the buffer, sleep until it would time out.
+            # note: If the transport is ~idle, when we wake up, we will send the current buf data,
+            #       but if busy, we might wake up to completely new buffer contents. Either is fine.
+            if len(self._sbuffer) > 0:
+                timeout_abs = self._last_send + self.WAIT_FOR_BUFFER_GROWTH_SECONDS
+                timeout_rel = max(0.0, timeout_abs - time.monotonic())
+                await asyncio.sleep(timeout_rel)
+
+    def connection_made(self, transport: asyncio.BaseTransport):
+        super().connection_made(transport)
+        coro = self.session.taskgroup.spawn(self._poll_sbuffer())
+        self._sbuffer_task = self.loop.create_task(coro)
+
+    async def close(self, *args, **kwargs):
+        '''Close the connection and return when closed.'''
+        # Flush buffer before disconnecting. This makes ReplyAndDisconnect work:
+        self._force_send = True
+        self._maybe_consume_sbuffer()
+        await super().close(*args, **kwargs)

tests/blocks/adcoin_mainnet_1000.json

@@ -0,0 +1,34 @@
+{
+    "acc_checkpoint": "00000000000000000000000000000000000000000000000000000000000                     00000",
+    "bits": "1a477df5",
+    "chainwork": "000000000000000000000000000000000000000000000000010eac567bcfcc52                     ",
+    "confirmations": 504937,
+    "difficulty": 234668.9893690928,
+    "hash": "79c4194bc0debc2e0d272ff6bd3ec87536867f0ff58f6a5da2d99264529a1df9",
+    "height": 1000,
+    "mediantime": 1548124552,
+    "merkleroot": "405bc6a405568e1841201ba46c1b8973643de75bea20ecfa67891910bd4bf5e                     4",
+    "moneysupply": 31004994.9999809,
+    "nextblockhash": "836e71740b533cad28d72cfa5025993df6e69a8da5064b3d77d1ccc6607a                     0da8",
+    "nonce": 0,
+    "previousblockhash": "a7af209eccf55cec92bf06723c53072bf7a6b9396a678e97193ec6c5                     f45b2e0b",
+    "size": 470,
+    "time": 1548124802,
+    "tx": [
+      "df275d713fcf5e78e7e8369d640201d46736c0d2255e31ce45bd5aa0206f861f",
+      "7cefb3db05be979676982fb8dc46de4636a1bb8b4d060fc71138fbf887606a9f"
+    ],
+    "version": 5,
+    "zADCOINsupply": {
+      "1": 0.0,
+      "10": 0.0,
+      "100": 0.0,
+      "1000": 0.0,
+      "5": 0.0,
+      "50": 0.0,
+      "500": 0.0,
+      "5000": 0.0,
+      "total": 0.0
+    },
+    "block": "050000000b2e5bf4c5c63e19978e676a39b9a6f72b07533c7206bf92ec5cf5cc9e20afa7e4f54bbd10198967faec20ea5be73d6473891b6ca41b2041188e5605a4c65b408282465cf57d471a0000000000000000000000000000000000000000000000000000000000000000000000000201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0502e8030101ffffffff010000000000000000000000000001000000011cd5c00a4a0f2fac5988ebdbfbd50742fc6700af20becfb8c0fefdd77240932f0200000048473044022009bf9a4e9c7060c00faee38da9f82fa9d41c75323b096aadb34c7efd91e3ec81022053f6bed2d56733d71f9d788b4a2635bbd964fae0de94cd3d1af68fcaaa87e28601ffffffff03000000000000000000c0a681f9c0020000232103454747c0d549d75b317b88cf05ba547270c600a5ced65fd07a6a358ea265ad44ac00e990f9c0020000232103454747c0d549d75b317b88cf05ba547270c600a5ced65fd07a6a358ea265ad44ac00000000473045022100b53ef914ef998990b9dcebbbe1607f898c929a716ca37bc67dfb2e5bdac9edd50220391e1033a3f4f5ff13cbc1a28ee5be57372ff0e13373032b690c4b30938a4e63"
+  }

tests/blocks/aryacoin_mainnet_1000.json

@@ -0,0 +1,19 @@
+{
+    "bits": "1e0fffff",
+    "confirmations": 273456,
+    "difficulty": 0.00024414,
+    "hash": "bbcddffb1c4682adb326f231c88d35d82767c994c2dfa2883aa21eaced2e3178",
+    "height": 1000,
+    "merkleroot": "00b8324dad56164fd02fde4a5887488ec674e9c2e834852717ad85cc26e9e31c",
+    "nextblockhash": "29151d174aec0c9275dc43c16d7d6812e4072d184657849f7fbd9178900e215d",
+    "nonce": 72613,
+    "previousblockhash": "f754a281a009e4741ee2215accd1d13f675207bba31d53a4125cd50155f31c6e",
+    "size": 188,
+    "time": 1551036197,
+    "tx": [
+      "00b8324dad56164fd02fde4a5887488ec674e9c2e834852717ad85cc26e9e31c"
+    ],
+    "version": 2,
+    "block": "020000006e1cf35501d55c12a4531da3bb0752673fd1d1cc5a21e21e74e409a081a254f71ce3e926cc85ad17278534e8c2e974c68e4887584ade2fd04f1656ad4d32b80025ef725cffff0f1ea51b01000101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0c02e8030101062f503253482fffffffff0180d1f0080000000023210396b64d45b24986f2092f54f5c19a5dbe2800f3363efe44c13c018a8e74e614dcac00000000"
+  }
+  

tests/blocks/axe_mainnet_30237.json

@@ -0,0 +1,14 @@
+{
+  "hash": "000000000013ae459f3755ac98e6dd01ad8364849cd1bcb1ca4a40f81a7ead75",
+  "size": 223,
+  "height": 30237,
+  "merkleroot": "de4266afc54356d351c11050a01165adc0e6b339268efdf1c578374b4212a266",
+  "tx": [
+    "de4266afc54356d351c11050a01165adc0e6b339268efdf1c578374b4212a266"
+  ],
+  "time": 1521830841,
+  "nonce": 1653639604,
+  "bits": "1b29c9b1",
+  "previousblockhash": "000000000014fce86cc81f0ad3135fb139764183a645d08b582a251b023f62e9",
+  "block": "00000020e9623f021b252a588bd045a683417639b15f13d30a1fc86ce8fc14000000000066a212424b3778c5f1fd8e2639b3e6c0ad6511a05010c151d35643c5af6642deb94bb55ab1c9291bb48990620101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff17021d7604b94bb55a0881000001140000007969696d7000000000000200c2eb0b000000001976a914b4237f3423d7fadad0a5a4a8fd6f6bae250d59dd88ac0008af2f000000001976a914f8bc2102e0917b9537babbbaa6e15fe57307062488ac00000000"
+}

tests/blocks/axe_testnet_30.json

@@ -0,0 +1,14 @@
+{
+  "hash": "0000028293f2766564553f51157be0e4a26beeca3dca90788138ed39e39ec887",
+  "size": 344,
+  "height": 30,
+  "merkleroot": "76fd8de5ba442a656f74198b2318fed22c5710f4ead482b5b3350890c1a036f6",
+  "tx": [
+    "76fd8de5ba442a656f74198b2318fed22c5710f4ead482b5b3350890c1a036f6"
+  ],
+  "time": 1580757343,
+  "nonce": 1105135104,
+  "bits": "1e0ffff0",
+  "previousblockhash": "0000066f56529ae503c85ce9692185465af4b400182e35403f7b82b040e948f1",
+  "block": "00000020f148e940b0827b3f40352e1800b4f45a46852169e95cc803e59a52566f060000f636a0c1900835b3b582d4eaf410572cd2fe18238b19746f652a44bae58dfd765f71385ef0ff0f1e0006df410101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff11011e0e2f5032506f6f6c2d74415845332fffffffff04e783be5c010000001976a914b717ea41435ecd3522222e0b3afbfb8ea37a0abe88acd41f4c2a0a0000001976a914f91c9cd14b776c3a8e0fd0b12deca6d1976ed36588ac45d0301d000000004341047559d13c3f81b1fadbd8dd03e4b5a1c73b05e2b980e00d467aa9440b29c7de23664dde6428d75cafed22ae4f0d302e26c5c5a5dd4d3e1b796d7281bdc9430f35ac00000000000000002a6a28cf5e77ea4cb19c4fbcba0207de9f2107f560cbd9e56e8fb1da7a59f180ac398b000000000000000000000000"
+}

tests/blocks/axe_testnet_30237.json

@@ -0,0 +1,14 @@
+{
+  "hash": "0000004d68dab78b9482b10d97c06df8838c3a4d4b127f4bfaff8944a1c7e2c4",
+  "size": 416,
+  "height": 30237,
+  "merkleroot": "0297c882867576d7ea600c379a871451bf8b05233371f2162afed6d712b1fd37",
+  "tx": [
+    "0297c882867576d7ea600c379a871451bf8b05233371f2162afed6d712b1fd37"
+  ],
+  "time": 1584597511,
+  "nonce": 394664704,
+  "bits": "1e0583a9",
+  "previousblockhash": "000002efc4918c1e85e751310855925f9c0ce8c7c2ba85855d548dde547590ef",
+  "block": "00000020ef907554de8d545d8585bac2c7e80c9c5f9255083151e7851e8c91c4ef02000037fdb112d7d6fe2a16f2713323058bbf5114879a370c60ead776758682c89702070a735ea983051e001b86170103000500010000000000000000000000000000000000000000000000000000000000000000ffffffff12021d760e2f5032506f6f6c2d74415845332fffffffff0440230e43000000001976a914398e641c57eeb9d85c7f11e4974a3acef8f0959988ac40230e43000000001976a914a20d1c04b065c2f6785345a660ba3d0b58be604688ac00000000000000004341047559d13c3f81b1fadbd8dd03e4b5a1c73b05e2b980e00d467aa9440b29c7de23664dde6428d75cafed22ae4f0d302e26c5c5a5dd4d3e1b796d7281bdc9430f35ac00000000000000002a6a28f5cb8e0314b650681d3f95240fc465630716b1bfd826e3ec20b25230d4e41d470000000000000000000000004602001d760000c6e1a2a7da17de253be30ba0bff759fb0b29df7af43ef1945687749aaa85e19b0000000000000000000000000000000000000000000000000000000000000000"
+}

tests/blocks/bata_mainnet_1000.json

@@ -0,0 +1,27 @@
+{
+    "bits": "1c027c92",
+    "chainwork": "000000000000000000000000000000000000000000000000000457e92ae77396",
+    "confirmations": 1470603,
+    "difficulty": 102.9501356144377,
+    "hash": "455490f1892e605c38d214613911d86f13d1387f8e5360b50b7dfdb6d928f880",
+    "height": 1000,
+    "mediantime": 1431399204,
+    "merkleroot": "5c147d3f1dcacd9a558e00962d5d7e5232a8f767cc0626de6e4c0a8c05e8682b",
+    "nTx": 3,
+    "nextblockhash": "2897fdef809bc73b80b884a327b0ecde7334848e1d3fdba73dcb42e107024131",
+    "nonce": 91364951,
+    "previousblockhash": "81b124c35efd6d6bcb17ea27a59da44a3a47c1dd2086e6084c980a3d762ae2a4",
+    "size": 656,
+    "strippedsize": 656,
+    "time": 1431399496,
+    "tx": [
+      "fec0b1a557d5870be78f2159a2d9b2ea9d9a62c524d5b8ca873a3842dda35b26",
+      "8d196eb09483272c3c380e9b607ea34d853a034d4c653660bbe0b20cadb7f560",
+      "75b0a549911262dc72773a2c7ff7802ca54ffed082cdcde36877cee04c60e825"
+    ],
+    "version": 2,
+    "versionHex": "00000002",
+    "weight": 2624,
+    "block": "02000000a4e22a763d0a984c08e68620ddc1473a4aa49da527ea17cb6b6dfd5ec324b1812b68e8058c0a4c6ede2606cc67f7a832527e5d2d96008e559acdca1d3f7d145c486c5155927c021c571e72050301000000010000000000000000000000000000000000000000000000000000000000000000ffffffff2602e803062f503253482f04496c51550858000277d12000000d2f6e6f64655374726174756d2f0000000001d0000395000000001976a91410afb18a0ef71a8a6753066e6f58c8fcfcde411988ac0000000001000000013618a0825b6b7c88407ec09f353979ea5ab8eb717fe9c4080c277d7b365996da010000006b48304502202ded546a7630d5e8f0b1bc97d88ecf5113a1bfe5632256781e6b7a6e11309444022100fe2c46b17be6b95c222d88c19b537a2a80c494d8e25b6cb2b61d23fbea55e209012102fabf61b27ec61346d0c4cde192988828306899542f18d4f0f93c142122245364ffffffff028b579038000000001976a9148b593d0240b97c4b7f5bfb4d922afe527c523b0d88ac4a659725000000001976a914352b16d2a158a240428bb2c8e76225220af0fefd88ac0000000001000000010851ac02a455737e4eb7811cf43609700dac4bf194001468a0bbf68cf56b16fe010000006b483045022100b597c444a2c2945fd511b0afa6f0bbd62479274cfe99158b41c4c0c5d2d2046f02204de8e10d00257f03b4f484261afc47e16ddbc2963047339dfce5fc8fd32036160121024570e2d24e27e1b1474d724c4143f645f229171d0380fc10eb62be62b6392d4cffffffff0256d25314000000001976a91422eb0cf9da17719ce85df16df8dfddb797c1f71788ac0c53150f000000001976a914e2a0bbe2a46fab50a987a151cc113a7b11405ba288ac00000000"
+  }
+  

tests/blocks/beyondcoin_mainnet_143500.json

@@ -0,0 +1,15 @@
+{
+    "hash": "5276d5e76ba065c2f3ac1a02cf31ae9d05d75b3151c64a51194b41b9aa5c6159",
+    "size": 451,
+    "height": 143500,
+    "merkleroot": "8a04f00045f903b5ae1a378e02bada5d6bdaf0fd385da1719427c5fa0cc63b65",
+    "tx": [
+      "05df92547ad12bf278fe453b591322bd58695795a5f917cba46613fe10dfcf7b",
+      "1594dad49084d64c674f3f1e7d6f13615a3bc044b0ff576beeb6d68479f56e4e"
+    ],
+    "time": 1611371188,
+    "nonce": 8744379,
+    "bits": "1c00a30f",
+    "previousblockhash": "31b55b0ecdc57b8163bc4d5daf8d4a1d5c5321c3c254d4e4563d34858177f554",
+    "block": "0000002054f5778185343d56e4d454c2c321535c1d4a8daf5d4dbc63817bc5cd0e5bb531653bc60cfac5279471a15d38fdf0da6b5ddaba028e371aaeb503f94500f0048ab4920b600fa3001cbb6d85000201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff20038c300204b4920b60088106ad328a0200004265796f6e64636f696e506f6f6c00000000013855062a010000001976a9146ec63057741c13c298c1639fb2ac8922d57ea27288ac000000000200000001cffdc62fa8c75f01269ca466f08750b98d4b1cfeda9fce8ab3d612572eb5f845000000008a473044022027e1a06e9afcb5ba4fcb09f9ed24c6fd81f490583ed73099697cae828cbb556502206c0c3a45c13d2e5a897a63261a1f6f5418b38ee12d77055a77d78f19cdd24c8a0141045a39ddce002c9b80c7fc3d6136ab18142005a5c064c4178f07d8dd9e47c7a156c36cfd9ce8ebd4eec41ebd2a8000fd9bc8608b670861baacf86bfb31538ae295feffffff026c35091f0100000017a914127c418ff54ef530f577e9d8b58c6a7cc75553ba875c59fc0a0000000017a9145a82f4ece6b7bc85e581222e09efd483c055a667878b300200"
+}

tests/blocks/bitcoin_mainnet_951658.json

@@ -0,0 +1,405 @@
+{
+  "hash": "00000000000000000001cbe4b4add61d60d9a470e0958f1b4024f202db9ca628",
+  "confirmations": 74,
+  "height": 951658,
+  "version": 624607236,
+  "versionHex": "253ac004",
+  "merkleroot": "01119367f99b89ac7022a2de151fdfe5755e6ebd5dc793d88701e4a4c5df31c7",
+  "time": 1780112431,
+  "mediantime": 1780110799,
+  "nonce": 2521760026,
+  "bits": "1702068f",
+  "target": "00000000000000000002068f0000000000000000000000000000000000000000",
+  "difficulty": 138955357012247.3,
+  "chainwork": "00000000000000000000000000000000000000012bf97f78ea6b907624a3c0a6",
+  "nTx": 374,
+  "previousblockhash": "000000000000000000015c67a8b2c9ffe8350289b4a64cc44e85f4c6f3094432",
+  "nextblockhash": "000000000000000000007e955c891ad4f664f99eb5bd40a5832d617dbc6bd05a",
+  "strippedsize": 44645,
+  "size": 99628,
+  "weight": 233563,
+  "coinbase_tx": {
+    "version": 2,
+    "locktime": 0,
+    "sequence": 4294967295,
+    "coinbase": "036a850e042f5c1a6a2f53424943727970746f2e636f6d20506f6f6c2f657529387c9be15e106af10b0000",
+    "witness": "0000000000000000000000000000000000000000000000000000000000000000"
+  },
+  "tx": [
+    "fd09ffa77f4de17d5c10b46d11f4440ddfccd2c9613b30037edd6db96a014865",
+    "8434f459e0080890cef04205ecfa822ff9eba0fa1497780e226fb1caa067e678",
+    "45a8326eb58ee3eb9a30a3c7f5ac1b67e6fc0445fa80ac046d04df0e98f612c9",
+    "e8a4eef8e2a4e528ba49d937f9a28b62bd8aad8e1a3ce0901a58b9eaef825b64",
+    "97d37b81a748a13e5b2be9c4c2a1b63a2107388dc92da022bd1c4c3e9b5541ea",
+    "e3073ed0e962c7f9d82a488f17ab3c66d4896d419edfb21d5812fed7f44f7d73",
+    "8d2ad2ccfb0d584e13567ef021113ab6f52ebb0772a71e50ebb9467ba81bff35",
+    "2d72aa60b1362c2a86bde7034b6f4162c78b9b8af333255c2a56106c077f102f",
+    "b7a93f659152cb9aa30ab012eb0a3d0267296a5267ce73393947bfabe7260d05",
+    "d0690aa2c7c9460f8d6f4e615df2f27bd7cc2a7946ba439826a0446ee3d22200",
+    "93db1b7a8308f2e4255ed360b60494bd5894b39db5d85159f9e444dccebc7115",
+    "39b70b16b28118ce1aaffe26d3ce868b8a07efe39ec454c2b7184c036e14294a",
+    "e3ab2b3ff1cb898b551500b8dd788f38afbeea6aa70e0f8b6f4ef65a9a1c5abe",
+    "a0c6832849712854f0e211a1b1720701ec7745b1bff98bec3a9f4f598e9001c8",
+    "8604a5ef2276af37de62166472dc5e36795fa72f8f3c0596b6517d8a987701d9",
+    "7ce2ac5f2819c0a5a896e5450e9334cd6b0365aad0638aaaacbbec3c9851d076",
+    "b9ddfade0f1fd0f69be144910f3b91e15c20440fcd7bbd0ee52db82c542fae49",
+    "d79b4d8d3500ed3e7bd6e0e62e7ab716ee4e9c1d8f7b6e2459aba34720cb9728",
+    "ffd1d41fa029b61027a1945e38726dabe0e0043b3aec794311c3374534cb5d69",
+    "d39e641777ced7515266f49ce54c2cfe3a5c67290827d4094f0eb6cc8a0dc54f",
+    "84ea69b900fc524daac5e722aa586aadf20844654acf8468cd7815ae3a9dfa5d",
+    "7205b5d76030d3c03446de897cd47a5f43f6857692f1460366c02a8569b156b0",
+    "1230b1f98061c455fa532e6c234d1cccccf382ba7c7abeaa309d59893c32994a",
+    "c0b4719d567483a506da5b6635d52b77cebec12470ed84c4749bd3c39496a7af",
+    "916adb287435606840668d605937f82e0826a704ec2365860414b704c6fec992",
+    "c35425a0f736e99dbb51679702e83a150aa69202e6e93e2be5ce8fe167c776df",
+    "61eb1e4e06d29a3712fedc29a60a6d260bda9500d90cecaec1d3d7a22de3cb62",
+    "5cddc90f4f529448a1849b9d1ff21e2bb647bd41262fdce149454933926dc4be",
+    "b517bc23776e0fdcd07543ff0a2f30ac6cbc3c13223c37c2190bd1c7d4ddb63e",
+    "ff454ab067134fd805a67cd40dc587fc4e0d0c0df33cc8712723dc5d8c678649",
+    "017aa664aa54677614dc5e1cb2e95cead1e0876505825e809b2574288a73ac55",
+    "7d6d8d3755dd2c33fbe6a7536dd8c19cdff002a15a2a19251d16e7a03acc1af7",
+    "214010b52a47d1f6a5f9400ba05b84186f14dd269a7b13464210d652f432b214",
+    "0ab0deb4cb2a8231707250618f7568fa3d62d54c58541c4d7fe6cab3f95333bc",
+    "5df72165537f4281d32eca07396d6a6a60cc72b76260740074b7ef5a89602820",
+    "24ea3188bbd97dd93dde459642155da18b18fd0c5ca99c4714f499bda7b42f1d",
+    "2de4dfd9d89ae8703170accd44de690295abf22ae9c3bda83ec93192ed866e4c",
+    "94382cd8e1446f864429d372b20ff98076be79ea403f61fa071b4454a3a62aa2",
+    "4d3330a1b751abe86782fcf6e0b3d4d316239298e294ca82cfbc8fa8df59f056",
+    "e67073057cc2be39b87013a59609d385cfa64ebd998cbefb9f1e51050ccc66e4",
+    "b50cb6f42bd66dbff9de22b823d4f5fdf2f7413ba0282bc2fc32ff5be22b6f7c",
+    "e4e77442dea1f5aba8f5a7cdc011304ac1bcded4f37281bcb496b50ebdc9bab3",
+    "b5c123d2232d40778ab384ed6ddbb2b605a29ae58d06150e78898a02c6719cb4",
+    "06272f747c54ba8e4ba18304274ac256da954706eb0f965c6f9d4da1a38542f9",
+    "6aae6f0cdaa4d5550a4ed1ddf1e3e1247fd8306c885088f25699dc11e3554373",
+    "066c0eec317a73e82d65934ac778cff639beab64c01a58531a69dae447e32ff4",
+    "a225d3accda8ef947765366a8c5ce2d1e0dc2d053e01a51f7486d21b3f8937d1",
+    "62f2887bdf0b99fd7afc116e3b0a54bed3883dfd02316337daed4543d904eecb",
+    "f9615701f7f3633cd79b918eb87d07b7455c2cf801c5e9ba4a5df57df6c05618",
+    "5b0a5300d2de96fa9eac390e13cb1db83fe496008897fc4be39bc8ffb35cda4d",
+    "014cf6783eec2ff2b716e300a314aa54b253a12308cfd1c7284e992de0dde75a",
+    "459c9697560e9f27c8c3e7955f4ee6bca926bdd6b542345370eb74f63a5eaf6f",
+    "8d6e2f68235e51ac94d6a54885d0eaa57b3b523e4fd2a0499d4271ce2a77be38",
+    "63e4aee5462bdaacc558000781e1ede11e54322d4e33c5f37037e6e25c36ce6d",
+    "768c0855389c62fa0af0fdb72cf14d969022171d2fb6c500a82bdcd05d145382",
+    "7c2915d47fc193d917658fffd6f8a7385e963ff03fb39e2e26a76295e6445ab8",
+    "bcc4d602e8ad39f7f58f1c9959fb6c37e5738ae829565d05d0456ef361d0257e",
+    "710928db8b7e6c4e293b276e5060ae55641d9a917cf652a055e2f6e5db1617ef",
+    "4ad7ab9b1bdaba5fb3377dab73d2b1050e6a2e61e5b93b71ecd5e09b62525ab6",
+    "a2db7aef1b371607494c987fcd3a4ac972d3bf9617c59b8077cfad782b8fa005",
+    "62520b8381ee25f7dcaa6657ee688452f22d8ffb23926f82cfb7a2f89eb430ab",
+    "372e98ccab389e888643120e8b78e2708fc4600ca7bf556c8b2d83bbb2d5ca16",
+    "705eddb5308b7fb71d1ee404b8a506aaba7bddc950555762e934bb70243f9762",
+    "d749e48d4fd50b1b57faf5c2c5ac80f57fb650d47e38497f85581e15fc8dc12c",
+    "b15ee885161916bd3bd62920e2fd1c8c93aacdfd022e6040adb2c0c57c0c7409",
+    "d5803468c54416834605814feb6fd16a5d88de3152fe475c0d73bdde23fce77b",
+    "47f92fdeaeac6283260c89b57b3a65ac100a016075642528c37bf8ba6b5879cb",
+    "05a73a5bd4ad38a19f8ca2c730dd852992a38d0621ee448aeab8a47e7619be0b",
+    "7cf58d5fe1435f87e23b6c9a37b5a325ebf1c3465d9a91e924dbb3f275b0e516",
+    "7ac9641d47c13ad037ec8d00bee8cd4a2d70ccd29393efc03e32881be3028ed6",
+    "7a85cfa16c7b8ae5845ca6a761d3de848eda84d00098ac8bdf2f935498fa3524",
+    "7aa0899734a1015c1ffefd45cedd8e5c86a1de712de0ac7ee3450ee53d8e6b17",
+    "cb941e09ebc1b68c5c8cd341a8f3c730de7d3041e4a68d8ca45782d4ff688c1d",
+    "aab1c807d1fad5b2fa9d3602956f7fa0ce4862a2f80485a0916091451a6cb7f9",
+    "3023980f689a50d6bcfc3b0bd918ea5a3fa1a950859077dea1e33a779abb7396",
+    "4ea0f44933c7a6bc29d70e5605bcbbafc06d7939f59f8dde2fa9546e68595aa8",
+    "0e440dc328c6752eda4aadbcc81c0981ec4982ceb1449f16532fb3cd35c1905a",
+    "adad0fba9374916afb2e5c44a682aceb1f462721d5d4827ce6f90bf6270df151",
+    "80d9130ff102eba3b0664b045331cfb103a9229315fd3c2bc7b907442723b926",
+    "dde0a79c1dbcb9960f8f396257ec0f3d7ac70cf63836ebcfeb3cb22d43a17cdd",
+    "8a9a4dafaf240f7d557772174913959d23b900b4f353c2549b9bad7a27b63e33",
+    "6b67a2789821b309ce629fb59a94cf08d6f83eb339974a3c5adbe674f96382a9",
+    "6df4551c85b2305b244c753b702d320347b84b7bba9f692c027c6bdf5ea0b968",
+    "8253cd57fac26fb30adb04504cf96f069f9f5e67d735e559b4864a9cd9a6d53a",
+    "20f06ddb07298e9b510594b7e685025878f3487844f871f5d219877101a51be0",
+    "d49ac6dd6b38aea742c14be850b203cfa4d72f06ae5b1db6a323c064e8a81591",
+    "3fbe0955cf5cc284add39eaaaf60c4ca6ed35a794a61d2a305bf54d988152e42",
+    "bd4085c7682c2436da1509ee23837f47de2ed06a5b08ec4442c2d11babe32e4f",
+    "188f692093f8830c845a85415047b7ba30ad35f6cbd9486f60c469622b15c94f",
+    "2c0861048b26c6707fc9b0c936c8d102bb73cc1f2506ec68b5ddb46e00072554",
+    "2ef999d8fd68160c9668b6da20090492df8b10d4a87c9b9cd586afaca7fbb78c",
+    "0e5cdab3e1bc03c8a98c3e1347b4e102d67991d14633faf25bedf7c0eee5b1a2",
+    "b67948e222f9436ac620aba98d6c08057f374e7b519295a464a060360ca5f0d2",
+    "257539f355882e1e68f23a5abb5b4f9db057d5061b475d8448c5ee74769e02db",
+    "c31771e4b658901defee4e28411ef93e9a5631d67adf7558e6a72bb5b8568bd5",
+    "be3f478bf819bc5224dbcd487db6039e48aa6552517fc3dde8cf773e1a67d562",
+    "cfee07af204f30638ca90bea1bca6ac3aafbd1d918645560019bc0cd19ec938c",
+    "0c6c096e80ebc3065ed22e425595fb7c2769d477697e2288f8b17799c1230d63",
+    "487d52ef8caa6db69c2ed287d6d6e2aba671c8553584500d95c3d3c2e101fa64",
+    "ea4690fcea3ab2d297986a286a9ce2f1dc1228f1b352aa1aa04d9cb1cfb75f7a",
+    "178dace7cb1ba3a804d7ac8813f2926a36f2c25c587291ffbd71773028c6c58b",
+    "07b7f6fe8002796aac002790d62aa60ee3e3133ac7d7cdb1e5420f198791e587",
+    "601e680d14693740730e62e637bbd643da4e23392474b5904c93b62bb32b60b2",
+    "70708252ae7517ac3245958278be55e5bd59a2ba16321b9bca049556a5106073",
+    "be6a913635dd9a795a67a5da40b3d05e43982595a030b6681293de893fe32787",
+    "6acc5a35d699ef2f23cff23f42031c397f2baa593595e13568a492906b31d07c",
+    "f2496cbaf3fa4a8aa58141a5b3888aba50e62e2f9bf209b66148ff1e1ea84fee",
+    "af61f2b7f1c3194270b541ce7f64a9acac1eedcc5d3257a0ea38e4b1869cb683",
+    "e26044361da7b3545dca06f463b74c53715b9598ae2028a19d9c2c368209d79d",
+    "169c95134d0bad226edd337d4e1ceb65591f5dbcd174d93784c1ae3889db828a",
+    "c03f26b2fc655d593dc7f88320ed7ffe35cc7d959e6819fb5a4dd8dda4d5408c",
+    "232565234bfe509b5256018bc3cc4324a4da545dc75956389380864486517dbe",
+    "5e2d30833e7643c9aedd8ab1ad4757edacd034d60b04eeb450e7757ed904cfd6",
+    "adbc7ce37cd59134773657f769bfbef6cc859a7894f68744f980a3b42e315dec",
+    "bd240ad219863854a6de76cf625f7659eac7cbdeb3cdc1643719910b88fde19e",
+    "b72c6f8b43fb9bddc02ae67a41b55d9934bc0d1cef83c888b56ac6375bb0147c",
+    "909a38cda2a1acbcb05051afc8aec66e94b91832194d9a7ebc81208b7cb57030",
+    "9f1c42cc7dffca8237601a9e9ea30b49877e8a3c684e8304d4d1650707d4747b",
+    "652dce84e2fae1d6d030428ab6cde2820f6e4d230924c36c12ef66893ee7f0aa",
+    "6ec0ac0f5eda13232e4d9212fbb9393d55579dcd73b47d43e5c71486b790a7b4",
+    "6716f5e881bdea5f894e63f4c84d79d73a26330a77c16c1ab1e285377e1b756f",
+    "eae81dfaecdd1d1acafd0d8dd0e2dccf20219d9e0bd4b0c4f106b44d0b9ba51f",
+    "585ac55b8c90d044e50188810f5980be0e52f41e26e56650081071c6c5f1e07d",
+    "fd01997a3786cea5ed4a22ea80990970974afe85270eae9595e54ab6a2158bd6",
+    "4f607d9c28a7c7bc31cf9dcee98e3f160ca9816ef471d43a69a771a3d46b505d",
+    "bbe87909207201a428de555ae56c9a6201803d5d2b95e11ffa2a109301c64504",
+    "64aa92d450676a52742f3d045729adcf851670b97a3e79a2107e316bd986466d",
+    "efcee007137302c75ff8670c1892d1ded6518743602747b09f7d0e2ff81b38c4",
+    "f50acacc98f5681ce674d6da16df46d29564755c7dea36f92d478f520d952bd6",
+    "1e43a9bf0e9868a27188916d84d60e646f2933a24a162c29311d9f31fdaaae29",
+    "f967fb7a44cc7428d3ceb129aeb2e21bd6db7b924ac5c699d34454e3085228ed",
+    "98829143eb246be7d36fbf104d0331fa98e77dd2e3b36bbe1d2e447fe86f785c",
+    "8a97b1c458347bfb964a6603b6eb396357c876a2a6983609a8ff84154b1094d9",
+    "048321023f2c9b74925783cdac9d8ce3de38b10d68391ef2884c6009bf3076d5",
+    "3838d122aadbf7ddae16bc9500e7b9cb96534e5fab11b0b5525914c78fb73982",
+    "1784d0f8a04decd3831bbf095b1dada4e05c408262bb77c3903a722e92945a57",
+    "f182a82d8a36f15fc9733aa0e587e2dcc2859b01590dfed7c346cb1eba6c8775",
+    "b313dc0be1efd2ba7cbae6826c203cc3c7a359268b21cfd3b4b7ff09a6682c7e",
+    "a4dbfd46e16cfbae1892a9f534d9141656714ef1a210430bc32c79b571416157",
+    "4a83e243c3889b08070457b66e47312169f818a6eeb389be6d600709f787d36f",
+    "64345e0fdc8a38b40398579e9987644f7fd16c2ff763f13f4902a7f9b4dcaf61",
+    "5f35cfea7909af1bbcccf643724ea9352a91dcf131ccfeec754cbf4c35deb168",
+    "df64c7790e7bc65382bfb469c14979f3fe19b33e2210619cc6395b091b1775d1",
+    "8e759e9635ca157ab243e572ce7077729be0e48000a29f7b53374a7d1fe5df06",
+    "e61949d24c41b2278e5e44b9892b5fea8d6acdc0889a9f89d8d18c685b708c57",
+    "5b7dbc1dec255e62fcd21af7ff488a665e5a92b65db088c5d9fa95c48e95c788",
+    "ca79539a8149e06bca3cf041b3faea836364c7f75f0939a94a17753618e4b1b1",
+    "8f83dfa814b16169d2e11369abd8fe99abd4b29fc5d74c11d1b551e76cf4b509",
+    "377f2302c83cadc182da6e7f13d69c1bf38abdea283e899bfa04ffa314dbfde2",
+    "e37af75a315074aa284b1c74c88c20f423832880317da6413cbb01c133bc6447",
+    "d10df075423ffb8c6993d8b5d8605763c421e4371f4587d5caf894364a00322f",
+    "dbd530717ff3703379b032dcbffcaa86e56316ee5feb595b53d5ae5cff81402e",
+    "3405190e61bab299a886b8fb69a27a66a49e7e05584347b94744d856f037dfc1",
+    "dc5716077ef0c39bf98d19bcc9c92f07298542fd633936698118d3be442ce94c",
+    "7780d060cccbd3d0f27be0938bfaf7fcb667ce8f4b675614ec3dbc8ef094b0b0",
+    "35b94244ba54ad13fe97eacf0c7f5a7b7263d4afe1ba884625f5a649112efa33",
+    "418a09bd7d94215a9966460c8dab2f00073edd4acea23f37bfb886eda5765fd8",
+    "f3dd35284c932896621ea26e7e6c1440f0b0c999afe0cb9fadcf0257772270a4",
+    "f27fe79f796656ad914918aec2948f28add936d7adc09fce4b4fa231c65960f2",
+    "0dccc434b26bf655b9149ea2f6a6261cda0f0a208324b02561adf844801b44e6",
+    "eefd01e497efb97e183a4c6bb37f46742763cfa26bf4a90f529533fb2af44b9b",
+    "815ab6e812e46e6f088065afb35ecf794c955cf9e65f029bab8fbf3ee06d03d7",
+    "78c71429129f6e3a7a1c67777359652be9afa55c1c21529790993e24522d7f13",
+    "5231553e0ddeb33ca6d3a26b076d66df9351cd5af6777d14ffabc3d226a33763",
+    "96996079505af4c33ce1fd33b59d8598b9742dd839680817c4d15c416d120220",
+    "8618f571f4e1fef1a37ab9c13ff34b9e9d58b89fa0c03407a4f690d0c7c70625",
+    "59d66fff0742678a2df323a75db9725b8624c30a6817c2e9699b30e06ea9ab08",
+    "e1575fbad9a17b3a4f50f1a39d2540b6241c76ee2f371df3f20803a4649ea84c",
+    "eb81e086a4ecc353f6af509f83e234f42963dc97b8526ef6ab70b68d402055db",
+    "1847af5acd53f8546b6aef2a11742bc657e0a946ba55802e04d2732adcbacad7",
+    "b46cb8b1f688325402cc1158e94a9945b81191a91a15a14e278fef3078c38450",
+    "5f4ed943b113f3f0b336a16131c49422fffc9c8d029373f8e3114d4e0d8d47a8",
+    "ebb203e941c8eaa94cfc9f922ea2b5bef3e88077bf7bd519375d375b23b7bd11",
+    "9507383a244dcae9084efd2f0f7d977e2f654702f6ba8fbea807079bd5f1f1eb",
+    "987a7ec5a3bd3498ba1cf49bf1c7e7a14857c068333a8936d5cbdba58949d928",
+    "de6e26f1be87ce4af200478ce87bd54a1587aa7e0950daf9f58f9289f5f8fb0e",
+    "92f4abbad6c86ac7502f0f65e08d94f01e0bcfadca33cbe8dba062bda37ac29c",
+    "302154927bd56aa614187997ec10d00f9ca3148f7866d9804477ff9f0b57ed15",
+    "72265b052549766b5ca398c06d7bc217d04fb7db236ee195712f3be008d4c598",
+    "2ed937d064143fcceb44561fc214f5090ec663b980a50f2645de1192f135a5b0",
+    "8c912cae4bdef7fcd6a88419d9cb10944c26fd4e960dd870768add7e27278f3f",
+    "ab2921eea6cee4f944d363d9b18071574e026fe871a971a53b31be6ae41baf32",
+    "54a74d1fc9f57eec1da3e62dcb9c9faecab46fd6d901b01eb146406ce24e731c",
+    "a225c2a47cf6c1f7b50b0da99cf9d3a7662262ac704239b35e8c5f33807c692e",
+    "f777441be1ce099fae469f4648d07e71c933c519da98546fbd178af5fd8f0722",
+    "609ea4ce347b53141dff5b98c4712e84b163465244831c95453cb45ad7c12f2d",
+    "4ac931b9090573b83285472d9f0d3cc4791bb00c015d7dfee1fc6edb487fa987",
+    "8d11e299cf6d145b69fe478e6384746033f98ef10fd0b4545947d61307c4cf90",
+    "195a2ae1b4c4a3bc366d6abb96e22304b593878805248eb4fe5135d3a1065bb4",
+    "5238ea3762e41579246a7cedb6551da30252bb094d7c0f3bd9c0cd7583697a4b",
+    "ba40a88f8c139c40ffca1766b2e702d945014bec0f69028ba28c131ee20f493d",
+    "18244054135ec5b8fdecc28c886cda6e43fc5d44467fdeec111fbef1be2ca7ee",
+    "8fc7b68b53d1dbd004498c2d404055ea3a8ccb4726d6e27103b3d623c3128736",
+    "ffa8a359ca418cdc83090a900d58daff19f939693a519b3b8a8aa080759129e1",
+    "c40235df8602a2761bf9a2f2a956770cf489c5375aafc37ba36c6b0f7889d958",
+    "cf7fb21888aeadf09a73dfde6f5530d05fbce200c68d11fc3c5ca5091b09a1e7",
+    "55c8df9162ecf87e377e2f887f2b6fd1bbe38df4d2286ffe874da9a968149520",
+    "e6abd918729d48e927a7d29e7ffe1e87b80db58f632b72caa6953f6c0fba15a9",
+    "3716a6f6ec6d135406b7ec08497de8fafc20994b3c36a32fdbdf6b37ba001e86",
+    "4a1eb1e665e0798087bf45b97c71cec4a8d0812ec5323e9984089db4f28fc134",
+    "71ba6493fd46014518927f0c8fa7aa9a2f3ec990485d3e5a545dec99fee75c5a",
+    "6ccec17d4b309eb265527f29af9485a0e7c6d6bc3b78985538ee5eeb2b25eebb",
+    "955e35c2527d581092032b1f4ddf93809f0500820caef363c4a39d7345f167cc",
+    "44aee5aaa57e2f51f8a106d2f338f05cd0721f7760b4c6bde9a9ad7d6fc12eeb",
+    "05b5161d06eac9bf1c1ea373b38d36c6b20e09d07d6d1f8c42be44b5c1f99eda",
+    "c306c9d1d7a98dfbc6d7a1d80b209e74ee2b31e86814d2a6b0c476788d3170e9",
+    "7002412b97b085f25fbcc50e950c3f62b1be8b511242782f839a0a0e5752b4db",
+    "0dd26fe38f20b47bf53e8bdf53e7e7383703ed62f33ec3c08eef301239b63090",
+    "c929e9882ec7bec6727815025e4a7a8583c492d1ecc0a798570f1cce09be8566",
+    "4dbe9d7e1c8b540763d2f461dad985c763a36d641f41afe0a070ea233578243c",
+    "f07d8555e4bd7a85dbc9b1fa260935b692e34ed9fbee4f9bfe7598bbbb28d3e0",
+    "dd54f4f0884513c6335a5d19ef8462f7f7e8963adbda34986c1f9c7c271d05ca",
+    "8c2741fc4f26440c8f6b4ffefcf1b1d1fcaf0adfbc4af6de7544f425a196599a",
+    "6160f69904a422f3592c26ca019492a3bdf90e35c1ad40e838b970db1fb75cfc",
+    "ecbc0061e1ab9deac27200ad3b6caaf561ee30f560427b83a76b3d9b7b6f0a3a",
+    "acc56067dbbef0322429e469b412500e722f16f7096891ede54e5799ce063fe3",
+    "863f04a99918e79af0a93d7e96ef2c2b5b1bc06340fb153d63ad6ef9eae22cb3",
+    "cc39f540f24473fc04d11b0135115eb05867839fb2b075bf58914dbdec0ed3f2",
+    "feab89427bb8ae5d646821f6b40abc20454eba69e55ce42057cb369f082a1c7a",
+    "571fedbd85fc80c54e7c5c1a9bc7ed9285032975e1179fbacc18c94413f65475",
+    "0ad7f1cd07d10aa23344173eb311e4b21bbcbafc2bf7ba55f6b851ad37fcf8a8",
+    "a271d672c9c69e891279abf0474d0d4bbfd4f555a5817818bbbec1c633948a0a",
+    "63108210d728784eeabaa1bf7f8fc7969ada187bda2ad3495151a4b079c3ee2a",
+    "a357b166b53cd1b5920c9cd9b8aa3caf55e5ec67c361d69ad25b4601d403bf4b",
+    "afc59848dd01431d5cd627fc58bc04cd010ff2e03eecc5386266d3702c739c52",
+    "68e72c982a64091057221a922d4a4b477b1aeb08967850fe51158f0ccae24167",
+    "4ca358243144db12cfa655289de554d565de0f745c4d41a907c4d1b42086e784",
+    "ef6815398d1341a04d8115799b71a85ebe80dcae7e415039562b1940aba1093e",
+    "737fe00ed83e3ab569c9487b641e4ea4751814fadca327fbeb224aaa1ba7a274",
+    "444cbef8d7fa73dc854841692642b7c3113c79cabde0e76ac479eb0c0f1010c4",
+    "067a6c429a877cc6462d97030f19dd0ba3b68e4c417bba6ccda82929336465e5",
+    "20dfd9fb82a54529e9869389c3685bcf1f6d8ee134f02bb1cce6cffbc324f944",
+    "406f130d11de90310b90e935ff9acf33af24d6c3927bd77063be40f443623a7c",
+    "24820ecfd95d05768e3ed6776c2d8853f2fb1d92fafeb759c60a77c39c131dbc",
+    "bb12d28050ecd2ba6205a01b429ce5fa1ce26fd3f54bb0fbf1f7b589788574a9",
+    "51418042b078a822ba030df4c66e1a5b37ddc52efcb575541041c2a83fb5c820",
+    "fb75e9d8d6bd26cce6fff1dfbba7f5060ec61860fb3f285b1d9ee9cf6acbb3ce",
+    "0e8488e4987e83e335871b0858e3bb4e5bb0695a4b77b958c5035ffafa128fdf",
+    "9b253f5cd29d9f3a3e90ab6fea77d19d4d4f886281ebdea0265babe804c78ec8",
+    "d30279aa47874b2db06958675dbaf328a0042833fcb3c40eb5569fae5c1c548f",
+    "5fe9e7fef8a6d5264c6297d3991327e81eeb5f7b7aa7512746f95ee4231c4706",
+    "ed31cb7ede5a3eaa5c24428660e4789cb69559f8ccc418b7fdca8690c85f24a4",
+    "ca0c3d3bdbdd5e5614cfc0cdf7b8bfbd5820b6112efa06dbc5e859566902d67a",
+    "8978867d2c9e2303a0e65536a34d3a5474be2b1227495908ae0598e2e8d97339",
+    "27a2a5405592b39e130778578d79900478fae8f4206021654c82a8ca5a419b4e",
+    "c3468c66cb819355bf8ad85744ebb9b1f55d6a977ba06f567090f25c5d1f9c6e",
+    "1a6880184324e02fec796c11ae9ed3a44c376223a2272509dd326458ad1af5cf",
+    "a2a4326863dae615b8618c0f29406f867ae40fc3991c8a9b45257cd1413eefb4",
+    "899e0c7d2cbe167b4902136c9c944f6b120903b8b03ceec444e1b55f5355c715",
+    "bc0038b2c313f9d77aa01ef5dbbd337cad311fc565bd4aa705b399442b06c1fb",
+    "6f429d6d55948c4380470389ad78c968cfb86c84c7e0addeae50044859b2c3cb",
+    "99c1b3997b91d56183c799da239cefaa7295409ee8dd2fa67ce0f22510af5eb3",
+    "e265dc0950f09cba1921101dcfc3326106842906358b444ef42370ecb245f893",
+    "4a8da5e137b7cb4f59f5d590813fb413f98e2db20b225c931427be824e6dafe6",
+    "66e6e1d0dd335eeb027dcbe20d4d5b8e9785baec841ea1f0c2ac86f94e7330ec",
+    "3bb8927abc7e85f7172b39f9e2c0bcb1da6f544d767ab64b01e3ee420f7887f3",
+    "6c94ddaf69e110fcce55cac64bf9b7c2569323c5b8f2d60446a5402f7d3ca110",
+    "430aea6fd36e95b6c74e000a92fbcf95ff43dee1f04c64f5a9fdf566d28221d7",
+    "d1576b49d880dae8973f5d06f6fc7049d17a6a962f102144896e14700ffdedd7",
+    "cb7587b036e951e6e837673029e4167e622bffc623919e9cc900f3700612e7bc",
+    "476d8628217c6e8f0b6b16503afb330ef64e8667b946de78520646dab384f040",
+    "e3251ac3f66c27109feb6457b9ae35a3453eddb8dd1e13981b70070af8b9b0e4",
+    "c1c7ce3a10ef4fc4092209b33ab78040da850c768d4a377251b058c333745fc0",
+    "85b516f84a0b5d8a7b17b7d4462ab272fbe13af6b90a777fd79ab99fcacca750",
+    "2bca6a1f81bae5212cc0b008e334d602bec774c196feb8e5948e773a331bcd2a",
+    "357a022aa2b60c89e37efc711db27cbe70dd230b145ea617b004141b8c9d76c2",
+    "c1249bcbf7bb76180a98bfb42bc8ca270e8632f536e1c1cc3a8f72962850b63e",
+    "0e65c769ac126342af63be94f770796e2f13e0e328f3ea4fbedb209ac3cac8dc",
+    "dc6963cdd17bfd8c8f5b46c666bb47dac25be11f3273b30fa9ba3ce2d8652aa1",
+    "97008824366dc14a3d874756a1cfa508469cf2903a996bc2246476e50d0579e4",
+    "d4cb8e3894c6243bfae5e1a0dac72684cec921032b95b2c6b539059f2111e24a",
+    "1a8e0e5e3feb0a9818c695f5e526dcd647ba9188089dc8f318e52a44a5699a7a",
+    "b2859d06691bf64615a663ab072c813b7f2a09be4ee752431d9233de8d27c37b",
+    "b56fd6874f1f7b6dc6924bf8510b162037dcc019511056aa5f6a759b9040f4bc",
+    "34a17fb08cdf5f604bd1576f3295da2677e899d273d73159405e7272dcb2fcdc",
+    "7cda20d8a5036c0da4009d60d4207885e3129fee8d9323c521b0c1ddd2a74510",
+    "88a8c25635b6f15fc101fcf781c492d53b55bb30a57ebbd7909a480511bb5656",
+    "30a6b16d1ba7e17bd8e8d3cc7778c46e3de377c58e040a15962a3b59cc6d40ed",
+    "5f579efcee74ad34fd62820b3d9d6683040092e1306cb938c4e7d7d78ef4cbd5",
+    "be0859b2aa22e26dff4b92a3df0f65614b0c2058a1fcdcf438c21a919965c5cc",
+    "75efb3e35c30a542df24540ea385271c93643cf39037847ee50c40cddcff2400",
+    "157ee7983dd16b2708afbd5af2a58c2b46684cc20b421811e38c81d7dd06ec01",
+    "a529775d5b281fc416d3ada2fc3e174591d4a54293c62ba0f6b7892a11a19603",
+    "2aa026925cde43d46f9ec580d82bece022e67bf63d68b6b50996b69b9a7cae05",
+    "01d344c1743850532d53b363017a15a271747112172d3f11c9ea453522ad1106",
+    "5ff2838f4dee0462d6a97a4d7249b7e2f36d76746991010c83444542214a8b08",
+    "53f59d9647c661f1f362bc71245418d933cefbb435400bccbf41f4aa6863c208",
+    "598d919fdde4afa252970bd4f2ba2be5158e7bc3fdd4048e9c6415fc6d3cb209",
+    "9e509408ef27b3feeffc2e5bd5856a3d768d8168f1fb3f34063b1e744d618d0b",
+    "e7408bee19f646ad1d58ff1022ff898d05c87fb7fb1c33098c1b2f4b7218f40c",
+    "5d55a63514ea228f6c63be7275db0a3278558cece407fbdcdc66d24022b3bd0f",
+    "fc2cadbb77f1a98bdc307a89a763165dcf15b60adbdb8408665c40c251f45a14",
+    "9782cc02eedc69ee34768cf268fae2b62b8bed9c54fabaa2913d18a4ea455616",
+    "c9ebccb6b54c6c35eb008278845370b337b59ed379db89305ef1842236596c16",
+    "648ca2dc08ec8ccba8bd1ab80129ce23815dced4e383894c19cf989b0f3d561a",
+    "d8a21abe334fa43459310626528563155a5e791fc777a19b5752ffa178236e1b",
+    "80b86a694b4b1777f8f710fcb1cf4d7a840bca55104af47477c961c3bd8f7c23",
+    "201b4b47470d30c8ceaad0560a856e60024286b248e55179b2a80ccb696d9126",
+    "5c794f91851d0c3c8e4b450550e38cd7accf93d0148e4ebc803a1ffe804c2527",
+    "2038a2967f54d1cc57a85d3bf7936bc0e510b3e5298940902f75db055744032a",
+    "6bd15521144eea7e166d2b5e758ade487d10843a1134c2898764aab6ea28ae2c",
+    "161b0d7e765ab2e86b95f6b6e72f22077e1c09d35fe9b984ea0337f3ae7d922e",
+    "365d9fb3ff4ffb7348d493cfd1ce2c63d555f85003bcc065c1eddea92bfffc34",
+    "eb6b7acfc60a9137d0a9170b81afb807b6c0d751c8591e799b19cdb2ef910d36",
+    "839b8222c36a2ccfd421a9c9bae056af217c1d74f3731cb28443c9b420404d37",
+    "d391e5ee2130044229d280f934e08bb4369bdfae7698215ea7f9dd77b6c77638",
+    "03db33f232b35af9cc767353dd041ce1dc4548381cf6a15b521cafb048910a43",
+    "2e0a7fd87e236cd45aa524d2419ffef907f5b499a78ad7e681c2e2c3972f5245",
+    "1cab574b274c854a3a69b3945f553631e205fc34a7c38890ca9fdbdd2b3b3a46",
+    "fb4345be264805587271eb2adcf13d4142ec3fcb092f9ff58892ff13e6e21448",
+    "781d4caca42a2caf92282ff37aeab7994049643758f4b2d5a62c081a3aeb524a",
+    "930c944338d0958545fb49021fb0caf2d63c3635bcbbb3bbb40d3737f9a2934a",
+    "af3c3635d38ad9acbce25bc3087cc2aa5e26e8c2909aab70a219348743ed924d",
+    "cd191f0fa302d28a543217b26fe558db6b086bb3277ca4c22ebad96bffe7bf55",
+    "a91f04a0dee6b863f0dd1598ab830176b887c622670c81fe4040e2fe0e31dd58",
+    "440785f598aa7b5d2398e2f160e630edcd1a20e46d812719f1dad3d9d3f1226a",
+    "e97ad3413c1ae1518bfc201c99d068c8aebb979f103be0a133f9d86d5b147b6b",
+    "7253de281247a6362b2d8bb28a6363f92a7fd870953e3f7336a0e83a18d3886e",
+    "660521875ff09cbd5baa48b852a08c8ee67034ce256c6b175407882a695e9072",
+    "034bce48896fc5f759f08785b4cf337b7cf352a931ce882746c6bf47b977e373",
+    "51c44302667be8d4e77e0ff6644967891ccdc934d40f4dcd26ec9f2b3bd2827a",
+    "8582c476b54884b9cd1b8ea945c066372f12786f044c7ffb7a25bb9b0328bb7a",
+    "497b8e3e988ec36a2aaa02841f43ec363626c2b14d61ea7fa56a05930c60937b",
+    "662a022b28600a6f5f93ac083165be6b64f87817266883e4edfb25222f96017d",
+    "ec309f297ebfd065148fe6d1a9b9c40efa0139f162046de97f6d58268c29b583",
+    "db0da6d1639bad6f6609033461e4a88462ef365275305add2c2314b0ae12cb84",
+    "cc13a2e6b00c94757967a70d796c941443599854d4687c89ba343001f983bc87",
+    "308fac3d6d3163fdfd0bcdf1e36938d175386c49e1b41ca2fe7c8b49c4ade689",
+    "630ddc91925e9f4ab4d5a03e13b529de4dc50a406b6abb5aeeb9b853d55bc48a",
+    "129c6a7d387b43437b0dcb86526b4aba57b52556a50a97c1e21d8315d9d8698b",
+    "59f6bb7f898c85fdde34925bd7d8c0b78fc3d1557e94b3408e938bbf14d76d8e",
+    "04a0624de249376d1ba1e6e61acbae647421ca7a9ce84d1854fb4dc0aeb66394",
+    "b7b63880b95671e36a1dc3bf52a7b7aa33b26c99ac2ae24f9ee5384409cb3d95",
+    "9bf4fb5c724ce6ab06e32327300d5555468300c109956abddfe04238c7d73599",
+    "af4374f2e681bd32a5c833440e4e38376f73620f9f32ebe07db2e5dac9837199",
+    "c52845b2910023e8f3a11b18d90b62d168aadbafd0b519d71328d35643c1579a",
+    "ba8fa22a4b35f66b2f64a4db849e1cfa7c11a680619ebcf80600ca989dc4399c",
+    "994740dc63f827fd7d2eb539bbb2a28b0ab60ed5b90bf55b6f3be5b2fadb389d",
+    "9380a7131c270c44108d89e40b13a4ad3a26f5b01f625cfba3704a9aee3a0a9f",
+    "b6f69293b69f1f18eae28d969dbb265afd2300eb5176709548da3605a1afd5a2",
+    "79ef484639567662bd26f237d4573473b3ec45adb671028f6c9342221f8588af",
+    "464a2ba8b961dea959f01648eed07c0cfc4aee402396cbef77332d2cff98d4b2",
+    "4b2882ec2fd9c34eacccb6da61230ea0d3880ceaf13927fcb7d86f7e6413ceb5",
+    "ebb7bb2c721d9cd7f92287e130a62f51cafb07343b3db488406568ce10bfecb9",
+    "dd88b4c518bf75ee3810ca5ad06729d902f0c77b28ebd21bbca7cdc61c28f4b9",
+    "658c8ee5446de601f40ee0559929dee2afd4218db36c2eacaca1e74d230b1abc",
+    "52e3a9a2e217b18a8997976537b1cca44de63e1b893d80a56ce40b8501ea67c2",
+    "f4ad6175b52b45a8880d39d7a6032b0fbd08c6e070074e4553351e72cf586ac3",
+    "bc09873a12259029c64d6f0180b16a25e79908feff44c20e35715f833b9b20c8",
+    "25ce7dc4076c3d6b4aa7d7d311ce4798517a88a82587f745784e31cabfbd8cc9",
+    "d6afcd51d7fd252ffd7dfe08be1f54a0cc0187863eb96d873c196e7931541dca",
+    "786b8d873700223c83dba0312b66577a2fc84d3c775d89643e70e9cbfaa4c5cb",
+    "edb8e17d80c698013b1c9599458e87daf81f0b69a78f3ea5c8d6529a49e5f5cb",
+    "6a726742c9d1eb33572932454e8570c2a3614a19b5cabc216d9d58cce4dd56ce",
+    "580cd2929607ee3ce90354ae7c6c129cf56f5328ea2a20296d073c4b5cd6d0d2",
+    "8e68f3b26a41c188b9e588e669ca8bc47366944011a15d06a5fe4bb32fd6a5d6",
+    "91dcb5c490f04944b4d99c0dc661650ba9cadf582396678fd2322fba4a8d7ed8",
+    "c031f8524c887eeab48c18f5f0a2cb23479d1d2041bb2aa55791f4c460e2b0d9",
+    "c4fd0dc846112dd70ab4e62b61c4acafbf2e7038f790412954a63a999210c2d9",
+    "568420fc34afef32efb3db1991862e13b3e9fb4b8aa5ef52c157540a981fbadc",
+    "6294bdebf5c455a57854d3f0217e1a935891846e60d8d49ca7ee9dd9000ca4e0",
+    "d01d688abb4d7e5418a6c52341071a7d18c9ea3ed32664d8fbf3deeb57aeb7e3",
+    "eab85cabdb2ee6fbb60480e75546c266bfd85a722adc9e1f820c0d326c8b44e5",
+    "938a582bdb3d0e015616f3b8ffd55483181f2a3fa006783f1cf6ed4cbe5d72e5",
+    "383993ccc2e7921c0269494ea1662377ca2576c8aedd161f5cf346e5eab724ee",
+    "931e0e96ede461d08e8019c09aed8dde8d4795e4375dc7180359161b9a4dbfee",
+    "088744f80eddddbfe87732e60ff738f6631bc93fffd67f5655ef8d55d0c303f1",
+    "613b9683eab3936325b46f2ed893a13b7fb7a85085742dc9adcf49e691d3def1",
+    "887e1d30a71d3cdd1fb44426f8ef0dd499413e6e9e13a18bc31befd3f15055f2",
+    "a0a6973572a7fa281c94c9a0e3314ffcd09304b0e62b474c34ce3e216b73b3f3",
+    "bc81284f910f03079128b1788c6a0fe1cb12dc05484b1a824f296939574e37f7",
+    "272e0fa3765652d82a92c0b476dd80d6238d2841ec687ef76b75c96d18955ef7",
+    "0367fdd4746a4612d562acedde4ebeb51e8806ae3e7a43724011fd3a6d8423f9",
+    "f3ee292e0726ff5724fd94d4072a7e48106d78b2f25df2660e2e245ff18678fb"
+  ],
+  "block": "04c03a25324409f3c6f4854ec44ca6b4890235e8ffc9b2a8675c01000000000000000000c731dfc5a4e40187d893c75dbd6e5e75e5df1f15dea22270ac899bf9679311012f5c1a6a8f0602171a014f96fd7601020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2b036a850e042f5c1a6a2f53424943727970746f2e636f6d20506f6f6c2f657529387c9be15e106af10b0000ffffffff02dfd6a11200000000160014c929f0c63bb0a162c43b445a72effea1b17432bf0000000000000000266a24aa21a9ed9f0a2e8aa095ed3051eb85e3c45577c5a1c8d1eae3683725a5b16fcd61f05d320120000000000000000000000000000000000000000000000000000000000000000000000000010000000001022ba385a3ab7893df57dca9483923a2cee7e97e6daa1d3fb916e6a172aa95f24f0000000000ffffffffd0d16543f48f1e45de1859737f1d6ebb36c3cc3cf84e458ef0e440b04b9329b7230000006b48304502210095bc2aaca8abbb02e306fae84648a724517310ff10fdce664c728e03c1d613fe02205f38e0658e45ede2daa334d9a292884ce1dbf1b5e6eec8522bd07d624981c0ab0121024f47f471d766f42c317bbe42644ecf8236f969a760bd7ca259ace283d82fca70ffffffff0135c50500000000001600148197413f4e53db94300ae40881773b4fce0f33740247304402203510f86a48e2fa501208ead97cddccf4eb92fe9926029aca0907ecb17875972a022001ed6ecede3937a70f4df3972faa2b03aaac25c0d1f042c71f65ce7a54579a030121021c094a4cc994fe82e2693839da4ee052fdd2482bdf8a8bc062237ab9c93b422d000000000002000000000101454413127e1e2cc076fb7f433593b7d073585119f1b2bb1354522d5edebbf6a50100000000fdffffff02975e010000000000160014541d0103eceafa490bd442782d6e4c35b3cc696cd86cc00a00000000160014a6c1c328d18548e1b13287a4fc91d6a77779b19302473044022009e929c1d65048853466f09339476c53af889cb83f6349d95d555c9b75c545ff022028a5dbc3f40813bc735c3b46c150352913b817f810360feb7012bd90b39074c401210367b45a14714638ab73adb9712c742dfe9b5be26ff84e86f75a05474105e926060000000002000000019d77ec566fc121b63204e87c389aae54dae991029bea1a1288ff6f1c557c9156010000006a47304402200bdaa55d224e6baf7268b1723456d579a36a636480d4798f552bba90edfdbfdb02204e75c5a0901ee9f02ff4080af27d4d42c9e356d969b5bf8a41d57f646fd0f37001210288d6a4c3f2eb77bd5797976eee50d3292e36313c556250b3fa065d6787e9f62bffffffff0465ba0200000000001976a91404c00d87715e3380b9f4b4406cae80f43ce9cf8788acfb6621000000000017a91485c6e55944b87104307e741b3e731e309891b20587a037a000000000001976a9141c2da16a554ac64931990ee02abfd5e6aca798fd88ac86caab09000000001976a9144d715ffc7dc72406ae93113cc8f04971f6405a7488ac00000000020000000001027b407438eecf8bbda0f611c176ac11008b7d04e853b544ff051b319920b49a0a0100000000fdffffffd782933e4cfc720a10637f38c4101f8e694b1a19f20ccdd143842ad10ddf2cf00000000000fdffffff03c7ae0a000000000016001419a3e244e359860491693c5f3037590dd624174076f106000000000016001402b6c186a88f10daaaad754cca4468824f01586c0000000000000000446a4230783334346334363237626135313364666362393864663334353862393861633934366434313361306137643830313365366333396633363130323530363136623302483045022100c18aa26581dbac105d89d22cadbfeee402220f67cd1e8ccb6da980f81581aec3022061f021294c52f20a1c633b0f27f115601d55f4d6999f3d98235e247d7fb8d4860121032c8a6cfafda3d9273f37c7696626f3db73bda9537ded6fe44f9c28ab734ffd6a02473044022034a6eff8724a0bfe5b2cfd6712185d8a673e4eaf46fcc330c31224628cdc263402203636623e85f36183b08c434790ec8ed0e07bbb342764889122fd9c5b60b5c5e40121032c8a6cfafda3d9273f37c7696626f3db73bda9537ded6fe44f9c28ab734ffd6a000000000100000000010104df769bf635b829e83a5e24945ffd1b03a7e6e7e4054a1589036069c021bdc10100000000fdffffff027c780200000000001600140dab49690a976a27e9308ab2656402a327b18df76397060000000000160014a3eb2e09dd5666bf2648ada2c758ad5f96c780d9024730440220364b1e52943a43cd3eec8569200f9d71ca8aa829ef0559a458ef682081e8e4c602205b202c565b0ed41fbac1747f6f35ab36db47216f0de5e9e4dd9bb9fb43b87612012103f2bcc5e686d3e9d3d949777de061374736ae4af736cb5548205a627464f9c5a40000000002000000000101582b1eb99547376c4e94362f0a9de966a793a95badbcb8d74bdcf94e6c74c4a70100000000fdffffff0398150400000000001600145ecdc7db200c4d199200c401146c2f9408e95bbf9db913000000000016001402b6c186a88f10daaaad754cca4468824f01586c0000000000000000446a423078663131366337333930616435366532376662383736366363393736346539353337383666613964383564333433343231623732616233663564313065633034360247304402202e1d25759209ef6fb7c341ea3984931a290f5003485227c3f9f80fa7625b827c02205d85791c41d3476772d115f91456b688e5766ef697449bb6b7e72c40045c55010121032c8a6cfafda3d9273f37c7696626f3db73bda9537ded6fe44f9c28ab734ffd6a0000000001000000000104567dc5f7a55b4dc78959eb52ddaddb6c8dfb2ca671ad2cd9df6676187bceeb110000000000ffffffff5fb64d0ce2b7311be5ae44382102244e910ec0e2218b175ad4a8a6fd0f19117a0100000000ffffffffaf02bf7ca574b8706ad5376fa4b0e1b5b5241fcf955aa0943fbe3f4c718b4b760100000000fffffffff82997b18b98e32de5f728d84c94699b75095156956fe2a29e521028bd8bf33c0100000000ffffffff014bce08000000000016001405c2e4ecf55c4ff7d807759003829bff4149a1c80247304402207ce4ef46ee4231766bed1c22b1b6ba596d1c42b86cfb5b03569792d252577acc022014a1ed83228a17883257830f064db3198bb01c82ccf56569c6e3c82b2cdc5b8d0121023b27ede1d4fa7c2a2651b7b2ec1866420c916e091d38f45e5979d5a97db0e65d02483045022100c1fcab6654883921d0bed8820e407f348055138899a6c85d48654e9e2012ddcc0220470149ebc794021682a3272acd8a83e3eb3fabdeb79a86cc385e9e56e2eLine truncated
+}

tests/blocks/bitcoin_signet_31500.json

@@ -0,0 +1,16 @@
+{
+  "hash": "000001279d712f44f617a4aba70521e9612115be3663cba9a3cedcdade8bdc43",
+  "size": 528,
+  "height": 31500,
+  "merkleroot": "935fb3ab834229151ed9999b5ce12b342a33d0121e667a9c40235a385242c623",
+  "tx": [
+    "4b314b9112077eb2c948b6164afbf246c773dac4ef63fd5e88cf35f954fb4838",
+    "eade0fce56112066bfffc031aa54dd186d12758c8d462792b7e1bbe17e5fc76a",
+    "ffa88d535d70527c5cf97ef074ba388fc5c0c098288c193fc73e0756539144b1"
+  ],
+  "time": 1617216417,
+  "nonce": 7764888,
+  "bits": "1e015ded",
+  "previousblockhash": "00000040f59599972481c6852e71319d079393f1fafad4acb1e4e5b4755c99c3",
+  "block": "00000020c3995c75b4e5e4b1acd4fafaf19393079d31712e85c68124979995f54000000023c64252385a23409c7a661e12d0332a342be15c9b99d91e15294283abb35f93a1c36460ed5d011e987b760003010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff03020c7bfeffffff02faf2052a0100000016001481113cad52683679a83e76f76f84a4cfe36f75010000000000000000776a24aa21a9ed3b31ecce6ca20fb18ddde6c3408b8b1bb44c92c2a906f106bda9634cac4b578b4c4fecc7daa2490047304402203424753b5d6e192026a09176012fdec097bc7c085bcd933690f792c9578b3f2402204ae2401b38d58d62c16600a645496b7a175549e6fd4c2c7163225094640eae66010001200000000000000000000000000000000000000000000000000000000000000000000000000200000000010173219026eba80ff9c7ca506376aa8b7db9dc4b82924838989c21b9ada48bd7cd0000000000ffffffff0106aac41d000000002200204ae81572f06e1b88fd5ced7a1a000945432e83e1551e6f721ee9c00b8cc33260010151000000000200000000010197250c31a4a0341e5423f281391292495901ed5c575a5ad22b08ba1515cc179c0000000000ffffffff012b25ed05000000002200204ae81572f06e1b88fd5ced7a1a000945432e83e1551e6f721ee9c00b8cc3326001015100000000"
+}

tests/blocks/bitcoin_testnet4_40000.json

@@ -0,0 +1,29 @@
+{
+  "hash": "000000000000000c1a1fad82b0e133f4772802b6dff7a95990580ae2e15c634f",
+  "size": 5826,
+  "height": 40000,
+  "merkleroot": "cad0d11c8d5e1304a36a882827e32801142d6a290ce1a2a4e8f0035bfa7f6d45",
+  "tx": [
+    "5c50d460b3b98ea0c70baa0f50d1f0cc6ffa553788b4a7e23918bcdd558828fa",
+    "fb4e96dad663f44fafbb1277cdfe2d7dbe50e03c99d527f5bff3ac9b4c3570fc",
+    "d991f3762c955ed327608c7edd1588c741e3369b0fe63d58eae9fd3959eef342",
+    "e451e0a76f1326d657895a5cfc4fd0be67cc96fdc7bb416b81d55f0b5d210c53",
+    "8bce727fe08fbb79f02682f71d0b1f33a79039cd7a70623a51945bf2dc86d77c",
+    "d1f636b56a494ffcb789682ec1db60d5be4100d3e6d760ab6f1a3ca2d7ff2485",
+    "285f28586b364ccbb35c0e092b6b16615f2cc31f3c1176ec52ed65b19153bf8a",
+    "659ff9dab924a0e1236e1b465cbd84195564ae297968028037fcab656faefba3",
+    "b86c427253faf575e49e54643962b0a8e2065289848e8616247a6b198225b5aa",
+    "4cafc859ac10dd7d297d7498d7635158ad45045312227b593b43b274e33674cc",
+    "3bce55005ce35f3cf632bbc29a215f7ccf2480194b7fa29c93b2d9693964d3d0",
+    "ca413bf56518af95872b305fb56502d465bbd605adaeffe086f38c934993dfd9",
+    "28771e00a44951c2f7d905653bc6afbc6dbefeccbad65c199388e05f54e127dc",
+    "eb948c9de926abccb9f65bb1e8c98906bd8b0e1103dc39d6a6f4e9b7b74143f0",
+    "ee75ac577088e34bd552b4aa3e7b4bf1b6528b6b395bcf713f44916aa9a5a2f1",
+    "3f792690080f2b03d7d5a0ea7e1d7d79d83242024043b685a3ab474f71dd4efd"
+  ],
+  "time": 1723908842,
+  "nonce": 1825621086,
+  "bits": "1954fa04",
+  "previousblockhash": "00000000000000124f9e271251c750dbe30d9b3afe9f06fbcfb2fe3baa160203",
+  "block": "0020b42b030216aa3bfeb2cffb069ffe3a9b0de3db50c75112279e4f1200000000000000456d7ffa5b03f0e8a4a2e10c296a2d140128e32728886aa304135e8d1cd1d0caeac2c06604fa54195ec4d06c10010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff4a03409c000004ccc2c06604389abb271024e1b5660000000000000000000000000a636b706f6f6c22506f72746c616e642e484f444c207c20446574726f6974207c20f09f87baf09f87b8ffffffff02fe01062a010000001976a9142ce72b25fe97b52638c199acfaa5e3891ddfed5b88ac0000000000000000266a24aa21a9ed3e249fa3559a30d20a840b05239f365bb7d93aa6cb5484f169ac8badea0e843501200000000000000000000000000000000000000000000000000000000000000000000000000100000000010143da142315ac08a9f9b04d456d51175c18951a6939c972382ac5e1e6a139038b0000000000ffffffff019c6108000000000016001463841fbc7aeaab5104602a378dd66e72261fef300247304402204974a564116d4537e140859d922ab821aaca036221e2d507ef92b96adaa6cdb6022021ab91e5c86e76c2f7ea4a6b36a3d5a5ab4643da808f1219d7d06903b5827a1601210204cab25185dc08801da7cf5d62e7daa92914d69665a22e03e9ecc30ea02fa7660000000002000000000104e404664445a549e21460911648e6c68b958da123ce455db1b6244bd28886d2320000000017160014ed62876440f9432d9f6bfc48d59cef0a24214c31fdffffff8e32e9d7a682a856bd8e6feaf70458c3031d6a6b64ca6047bd9ad3c066d350990100000017160014c9a8d47f08dfc91a863111af39291486f09b534ffdffffff04b2752cab3d1409ced16e9656985b2b724cb1a6d8c264fd3e0c44072b0902c701000000171600144e140ec7d2f05008e5f785fb9a058f85429f20cffdffffff8a6929fd7d22f5634971620b6d5e2932a7ab60cc70e5f0509d7260b5054c28b90000000017160014a735c6fb24355366304ca88b867bc3a6c8c4852bfdffffff037b710e00000000001600147dd92898c488a0838e5ea32b9fb57d2a49f14b3da0e20e00000000001976a914a0a3a6eab4357f6cade3b5e0031f13564ef9162f88ac80ca07000000000016001463bbdedda0703c995a632a1a4ee92ffe09af4f6e0247304402205028aba5260d08db3444ddeb8f7a56e1b7278c8071ed7e8d37078a9d9ea6033102204c9c15ba2bb2516c8365bda4118ac189f1fda54ed38a4d7002e5d7fa7d74c18b012102ec7430e29962705708224aa88169072fec1094badb1103213939b84ba2ea734a0247304402204f7f475ec40edba8fa12cd30e04752e6c45937a976f93f9076dcf04361513ae702207363038de2d9fc2de7744ee2e1cf7548ca6ce9979c81f6c0ece495ffa2cc646f0121027aa0626d3fff010cbc240a25c5806c717b70523e0601737d4645ce7a4acee88a0247304402201b1784f21a2376b3578f3a193449a86050edd59f1be9aa0da30e9c808844e24602205102ed3ce00cb9fb6d56e642c0ed0c7a460ed6be4592de8bec23793a0d8c72e30121036e4bce262c7b4520b7012d84424324c9b093b17559133cbea66acb798274db850247304402201dca7669b20a6cb677c2f44e9349bd96472a26aa09ada0d3cba68f77ad1bc514022075f309d836af6195c677999659e86d5e80e196c2a7ccda68bc30b1ce940113a60121034d2b3d15f3b7856a7c350748ae48223dd05287c682bce97e6b198ff4ab9b61bf3f9c0000020000000001017a18376793c140241311576124240c41027ffa1d1b1d3e64bbdadf6b4fbe4f910000000000fdffffff02fe42d028170000001600147b458433d0c04323426ef88365bd4cfef141ac7520a10700000000001600147d237cb1fe62153805fa1fca6cbc74765ccaeb5402473044022022784ac1c15c76b320274fcf99cafa55ba1554d0e742f79d5bb7ecd2f3725e9f02205f193c7f36742aa1f23b091d2191208256e33d6084eed65230f5f238432e03370121030db9616d96a7b7a8656191b340f77e905ee2885a09a7a1e80b9c8b64ec746fb30000000002000000000101b2a2b9cbcd6acd886a3adcc7e172d7cdece234a8f4502079ed2d01fa46322f000100000000fdffffff02246d5d13000000001600140633f9fe93dd480f20159602e4e07e1721b7e4c03e6900000000000016001464de410cc361cee4f8b506567df3ca9f30710067024730440220680ae471f2afe3dfd90b276e686e191525f8a36e1e87187d1742c8b5c1b17b020220173a293064b41c70cb58bb1a3715dafbc56fc0a786c15415c9cdda9b237a86fc01210396867c45cc97e85d704a9afdebb388bc6feffa4851dda31168ec2fd7cf5763663f9c00000200000003234711ca14f70dbd873b8606ad0978c82928aa9eaafa6ac5291b6b25789862f9010000006a4730440220028ac25e04a859ba1f7a3ea8f4bdc175adaadd8ce8575d53fa9db37d5ac153e9022046344c62f2e5ddce6ec2f30cbb2ddeb6495621297c645b4948093d6712b586d50121039f4c6ee26308a34cd14a1930fa60a8415378355596c1153295e5275dff66fd90fdffffff0ff3701d18c63d4daea337060f47ade86466e8af46aa918827cc63e3482ae937010000006a473044022025897f1f067f6cbd818fb114a89dd9ce27de00842092f9eaa45374b820f320ee022045670f35a7b15c7e4b519445460ee3cac1ecc93a3bb858a892c9aab46c5ddf7f01210300660d1d4fc56e0b8e203e6cb667f7f78c04fa012cc544686b16a16b9a1c31c8fdffffffa70f23cb877d1eb479a420855fea388e9039a26b5f408d30de59d28bd15e4dcb020000006a47304402206055c9f81ab41b1ed3933422add9949535103e5aba7a8eaaa3618ae85d1a4999022008d3fc627703e6778f6d401b10f1fd71a6f511255cbf5f3a89f9c508ef50f37f0121027aca30f737a95a53f3da28cde58c3517860f8a51f850eb699f7d2bb9039cd46efdffffff0316fc040000000000160014f45ea236de6fd4e2ed8a6e6320cdf2db173c6ee0c1040200000000001976a9147dbe4c26a474503bd6f576c7b36c31d37861df6088ac892d090000000000160014f66ac1d07ccc33d517e26f9dcc00eb6468709105e89b0000020000000001010bcdd5e36694495e4a4d2f746d66388f276919dc84a7bbc612b543f48b36f40c0000000000fdffffff026bbd7f07000000001600147b458433d0c04323426ef88365bd4cfef141ac7520a1070000000000160014989792062dc8cab87dc45f47718950b773ca7d3902473044022050398f98e96da6b4b8cadee89c1eb5145a7db044711b4880e1bf66b2b15e850f0220764043c517a1df9dcdc45fd62c381e94e654eece565cacc53bc5acdd882c511e0121030db9616d96a7b7a865619Line truncated
+}

tests/blocks/bitcoinatom_mainnet_586900.json

@@ -0,0 +1,15 @@
+{
+  "hash": "0000000000000000da3df1b019576a00d93974818b392b4c1a0a15f3df7c66fb",
+  "size": 510,
+  "height": 586900,
+  "merkleroot": "296a6318fb26dd39534acbe2f2c4cd82d0a9ce00d727d37deb0684748f7b73a7",
+  "tx": [
+    "31417678bfbfb9784494974a1b5a7fc5b072c2fc1077f7951158cba43452d0af",
+    "253b8c15e7d5449dd604f07b5923a59ea35c6d0db0c0603497fec51cac317533"
+  ],
+  "time": 1518611740,
+  "nonce": 2909803309,
+  "bits": "1903fffc",
+  "previousblockhash": "0000000000000003060c045d260d64f50cbd03f827630974fc7686d7b0ed002f",
+  "block": "000000202f00edb0d78676fc74096327f803bd0cf5640d265d040c060300000000000000a7737b8f748406eb7dd327d700cea9d082cdc4f2e2cb4a5339dd26fb18636a291c2d845afcff03192d1370ad000000800201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200394f408041c2d845a0877ffb4bff7f26d020d2f6e6f64655374726174756d2f00000000020000000000000000266a24aa21a9ed370b337f1e15c7e0159715ea1707ec27b4a123f1a8810ad88e3b24ae412cd764288e814a000000001976a9143c0b217ab4208ab437dfc49c96d7d6f9e6608c2788ac000000000200000001af692cd7d8192a3b80f405e528d4cb4bc49efcad44c6d290a11cd21134a3bb82010000006a4730440220721a2935cc808b96ab20685c4bff77a1c253c5827c9fec4e7a54f2170c44870e02203fd847b15b449f3154acad12ed2f1d29b067cf73640e678a4b847d851441f4b0412103e0d0b2ea7fe2b07a3e26957a995190e4cbbd590653c8b118e161bcbcdbf52c4cfeffffff021d1db709000000001976a914035e3987787b71e772366787e087c8e9351dd1a888acb6d84c00000000001976a9143acea32e9ebc212a858c28a9624fb8924f35d8f288ac93f40800"
+}

tests/blocks/bitcoinatom_mainnet_586956.json

@@ -0,0 +1,15 @@
+{
+  "hash": "27b7efad91a595c0717a2e9fc4fb252545000c18a6610130d15382be83c402e3",
+  "size": 541,
+  "height": 586956,
+  "merkleroot": "81aa679f1878a93367dad062c8a6964847881260c19320b4e3664c63deabac7f",
+  "tx": [
+    "789dda752eecf22056a93e8bcdf89eb31ac8102485124a7488fa0472250ac382",
+    "0f35c0b9bbf661787bfc920b5f4571440602e5723144c10b8fa5629dc3279abf"
+  ],
+  "time": 1518630315,
+  "nonce": 0,
+  "bits": "1d00ffff",
+  "previousblockhash": "0000000000000002c32c76bc7971dd10adf28abd4ff45c1b23c5b74d0265ffcc",
+  "block": "00000020ccff65024db7c5231b5cf44fbd8af2ad10dd7179bc762cc302000000000000007facabde634c66e3b42093c1601288474896a6c862d0da6733a978189f67aa81ab75845affff001d00000000010000800202000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0603ccf4080101ffffffff02ca000000000000002321034055f219c9619e18ef193c0def3e43ebffea827c950f0e78ee8f146b9f4c8102ac0000000000000000266a24aa21a9ed164fe82b7473bf82cfdd7c28f90668f9fe78200022bafe159f293b4fe5e10832000000000200000001bef5797f980f07ba3bc619d2eef2ad728737709a747c4adf477dfb05186e7fe2010000006a47304402201fbef43b1877a013291effd9d9da882d128109d89db28ddd96f0bcb65ac4aab002206893640e754a9d147355474229af237581956fa191cb806648744e2d6ba989664121034055f219c9619e18ef193c0def3e43ebffea827c950f0e78ee8f146b9f4c8102ffffffff01b67b814a000000002321034055f219c9619e18ef193c0def3e43ebffea827c950f0e78ee8f146b9f4c8102ac000000004630440220413932292b5e1cc5ae8cc019b11e5db7ec4432d260e8fee0f9c0a195a5aec28902204a90b318cb1b4f77ebd678db45fc0fce5ab3f0cc2d89aacaa523d8b6cd82f29e"
+}

tests/blocks/bitcoindiamond_mainnet_496659.json

@@ -0,0 +1,19 @@
+{
+  "hash": "abbed178fd5943a1583de2f1d48dbd18bb49d49e2498366a4dde02ebe8ff5e62",
+  "powhash": "000000000016a5e5f562ee36daba04bb50c82699de998d0e20ac08a3943a6900",
+  "size": 1438,
+  "height": 496659,
+  "merkleroot": "78479b38a072b1da6d84e53efed2a09b5878c13462d0ff6c468a2130c117fdcc",
+  "tx": [
+    "914ce5d8967e32cc443b445117bd0da38d1c110860cfe7f49644654d119f02e8",
+    "a407f6eb6709c67a60c7af7185d160fa8a9f88306c73ddf5668e9953df647353",
+    "e20d84055134cf6e6caa6b99c7564386086720905df1c49aa5997222ca8bfa6c",
+    "940cc720f6385b9b9184f68524226de013f706333f907d111a4e74992bb44003",
+    "0c814cc375b6486304159121c1a2f9d0aa708ef8d11eb044dd8c151a43aa27a3"
+  ],
+  "time": 1514019261,
+  "nonce": 2219259,
+  "bits": "1b6925cf",
+  "previousblockhash": "460390420cdf623a5a57ffb02ebeae15345b0324eb8150c49ae1826db2cb4fa5",
+  "block": "00000060a54fcbb26d82e19ac45081eb24035b3415aebe2eb0ff575a3a62df0c42900346ccfd17c130218a466cffd06234c178589ba0d2fe3ee5846ddab172a0389b4778bd193e5acf25691bfbdc210005010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0403139407ffffffff04009b8647000000001976a9145efb63ceeac14fe5d518a057338e3d8a84e459c888ac20f1fa02000000001976a914cf8c5810dbb0e77bfa5811ef99a509978d85dd5e88ac0000000000000000246a224d696e6564206279203131313131313131315a378b8e1a1e555e000001ffc40500000000000000000000266a24aa21a9edf76fed3ded3f7d6b60934dd6acc8bec321b6f35b581da0d10514c558b3f189e401200000000000000000000000000000000000000000000000000000000000000000000000000c0000006f51630e6f847f16bfef023c1987e890304baa77eef212a94f7a064ff3e0e72500010128b0c588366178258de1c074542c9b9a6bdd5dd0b5022fce1cf593ed976973510200000000ffffffff01583e0f00000000001976a914e92a30c6c5ac231c004c1b3170874b390b03acb188ac0247304402204d9bc16a0814f67d09f967191db2c04898ac8627498361aea19f7876a1abfbb702201a609c8cba158ca7c9594cefab8fb3505cf3fd5d061e8a1db0292983d071c38a0121027314fab69a06e2f79fd7e9f47ed780f45e6d8227d03ba98f79db9899259d1db9000000000c000000a54fcbb26d82e19ac45081eb24035b3415aebe2eb0ff575a3a62df0c4290034600010128b0c588366178258de1c074542c9b9a6bdd5dd0b5022fce1cf593ed976973510400000017160014371727d42f36c47f501fdaab485edb41b2134174ffffffff01583e0f00000000001976a914e92a30c6c5ac231c004c1b3170874b390b03acb188ac02473044022003402bf9997859d4a437b0c0d00b52095833aafa5fefa0d88cd30b5cc31ee896022064ad27334dfeace63e2fe28c01793fc7b5366bc2ad2dac30276a6ddb1e1afd310121027314fab69a06e2f79fd7e9f47ed780f45e6d8227d03ba98f79db9899259d1db9000000000c000000a54fcbb26d82e19ac45081eb24035b3415aebe2eb0ff575a3a62df0c4290034600010128b0c588366178258de1c074542c9b9a6bdd5dd0b5022fce1cf593ed976973510500000023220020b3bfabc6f5da87286fe387aa59397fcffd800e417e0e0c0230018fdf8a692008ffffffff01583e0f00000000001976a914e92a30c6c5ac231c004c1b3170874b390b03acb188ac0400473044022027065788b6d6e778d2092067edf5b13350c2b29fda474ee69450c1693680dd9902200b42c22cc9024c87dca0c8030d66cc102cfd994bb3f0c7804961cb3ec4d0fb2c01483045022100956ba661054d5b9ee04930c926b0cd83cbb0230629e23b6b1f4a45518778fd06022009cba7398d3e25eef19e704eaebd87ad9010ba11185233af993bf0c23b739fd701695221027314fab69a06e2f79fd7e9f47ed780f45e6d8227d03ba98f79db9899259d1db92103088359d8c4df1d3753c67946de474de56a717ca793a9f81dfcbf1c9a470ff01921021841bc49e8baaa73dfa26074ca766bd6c31e83a9169b53b5556cae446dce1d4253ae000000000c000000a54fcbb26d82e19ac45081eb24035b3415aebe2eb0ff575a3a62df0c429003460128b0c588366178258de1c074542c9b9a6bdd5dd0b5022fce1cf593ed97697351030000006b483045022100944916c41549c8dfbc213ab8eb10b2987ef2bed51ea1eb51cbf3987951b99d9a02202b5feec10cdaa7f09bf23e9d3fa6825e2a72cc52bf25266fb5fbcb63ea73e8de0121027314fab69a06e2f79fd7e9f47ed780f45e6d8227d03ba98f79db9899259d1db9ffffffff01583e0f00000000001976a914e92a30c6c5ac231c004c1b3170874b390b03acb188ac00000000"
+}

tests/blocks/bitcoindiamond_mainnet_497773.json

@@ -0,0 +1,22 @@
+{
+  "hash": "e75d236f3b48cc191035e1909f5c60ea1b8dab2aa72e1111d434022a64a4f74f",
+  "powhash": "00000000000a965cf3ce19c39bd4e7e88c40f263741a4bda707e06df881b2e03",
+  "size": 3667,
+  "height": 497773,
+  "merkleroot": "7fad129c169e3d351a4128d9ef0749a0f3b4cb8ed7553f9afe955b03eb9cebac",
+  "tx": [
+    "30d25160dd2dd7cc221ebfadddd979c26e69aae19ecc217248d14f4b74a1d10f",
+    "13b78ed8768e54c67c96075b35c4cd60f3c88d4b513eb46ace4ebe7d21c3c57c",
+    "a771a4c51bbb27b693d02e39895d62ce7a2015c804b98db3730e3b2b764bc30b",
+    "bd577a5ca685828d02db78884acb81aadd8f5bd4b11bb40c396bb376d6ab62d4",
+    "ef8655812cb974a3bb62dc6d59b2c3df5f61c0bfd5bfada87747502ee9e0c75a",
+    "21bbb19bb6ed84c2900d9ac8e35480a6ed6e843c84cb868dbec1749a2d659fd0",
+    "948559161f3fdb3574a9ba26be79b0b5f8c24a9494d935da9432fc2e6eb077ba",
+    "51eef39fccc5f4ec0804a422d2dc8d8e64148ed75837822806278600afb7ffdb"
+  ],
+  "time": 1514630683,
+  "nonce": 9829296,
+  "bits": "1b14ab5f",
+  "previousblockhash": "395e48a86b3de4d2866747847fc3f1301ecacc07a32911ca528fc2b69c716ad7",
+  "block": "00000060d76a719cb6c28f52ca1129a307ccca1e30f1c37f84476786d2e43d6ba8485e39aceb9ceb035b95fe9a3f55d78ecbb4f3a04907efd928411a353d9e169c12ad7f1b6e475a5fab141bb0fb950008010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff04036d9807ffffffff0480dd8747000000001976a9145efb63ceeac14fe5d518a057338e3d8a84e459c888ac90fefa02000000001976a914cf8c5810dbb0e77bfa5811ef99a509978d85dd5e88ac0000000000000000246a224d696e6564206279203131313131313131315a3f2b2a3402d93a000007e0d70800000000000000000000266a24aa21a9eda0dcf3669556e89aefe0855b16ae4064549bca3e44893b1a230421c45855d55401200000000000000000000000000000000000000000000000000000000000000000000000000c000000395e48a86b3de4d2866747847fc3f1301ecacc07a32911ca528fc2b69c716ad7015cef698943b2ad23bd4401df6b2e31a447c8868dd8758edfbc12f7bece25804f010000006a473044022025ac5501e32467fb003ec33c29b54b2facf57500269ac8e7ff2d6505de935bb0022047d91899603c1eff45b8030c3511174a29b79ecc63ee90eddb3b6ae0d860995d012103643e8e6a4883a3e3e3f7632ee568c66d7d54d6f5662486cabdafd880425dbd77ffffffff01301b0f00000000001976a9148c9261463e56899c4c682efc62401c6b59dde0f988ac000000000c000000395e48a86b3de4d2866747847fc3f1301ecacc07a32911ca528fc2b69c716ad7019ef4260351ca5a1d8b7d101dea0558e0f0869901a6ea4a5cf876d3ec1f5c67a8010000006b4830450221009fdf17a30431dcf26996a8c9669090e4a17977118140ff18eeda3d17815c4c7c02207e723788913ce185a0dbc428fc7a635a0e68a81299a095f3c9ec216b43b707b901210249638b15f5d391166d70c25379a9179c9eb6f0d79014459b6ca15506044136dfffffffff0167666e00000000001976a914664e618a8054c04490371a9397086e6079b4763988ac000000000c000000395e48a86b3de4d2866747847fc3f1301ecacc07a32911ca528fc2b69c716ad704448e488d23db775d9b34a345c8b7d98cc314d4f7919d689ecd8b461594a18937010000006b483045022100cbb22c0d7b5163d98f103523c43eb9159d3bd6c3581c4b015d15c80e53a118bb02203777c0a6491b5ec3ffb1af3e23aca76da62d74a84bcec34812da0d5d358602f2012103090e96dcd97dcea298cd2576dad8a70289250fd6003e11b98ed1c08bf81dfb00ffffffff1caa61f2839ec736f4177c8b0cdb051eb76ba9b89492daccd90892da5faf8108040000006b483045022100abe883e52883697a52ab445522cb80ed68cb68b071e47521ede3e1448705e8120220103e9267378943dcb8dc31edc2880c48334351f21d872e402e3d44c6ee14d9e4012102510670e19684ba236375de80bb992757ea9cdd03718e77f99157e045374be316ffffffff2324cdb9dcc0a05d1c08073cb7547dc84a760aab998f22c3fa2d74a721397331180000006a4730440220095debd8d6bb9599d9a1ef29bda8a58a37b8eae7a8eba01bf69d2ffd803228e1022046cd507a140cfb78b13478c817b7e022209bbfe6671bdadc99f1851ea131dd7e012103169385f6a59da7903cf64c0d994d7ffcac13e72ead2153c032e182e644d8e568ffffffff2e54384a7ff42018b9379642a48ddc1a698788bb87392d714d1809056654e555010000006b483045022100b7bc40b949e1ecd801b16eeeb83408364261d94b404562162fb5a3512949611602204d5bf775d96a9dafdaae7c96c2955584af307b07f4de442cfac4a085619522fb0121033e56a0a42a609c256cc35eb06aec01debf6b544fb769715561fa92776592f275ffffffff0131c4960b000000001976a914a0d0c2b99bd843055496ac8b67cd2006aa167a4e88ac000000000c00000033e6f89701b43f7526bed124b1c237e03bca4a43a96d1b31757ca8087653aa9d0210549e184f6f091dc6004a333ef2295951abc8e400d4b96487dea6e69b1aca20040000006a47304402207d254aaca5ecce5a1c3b60f87e6d327fdf3469a2272469ea76a1b7f347a0b91b02203a6aba222765910ccf711000f030e34a7abb9ce40e6eeff823409dfb0def4d36012103782ffe90d1a5ae5ca00d6220f637a92d5469a056975c564e6b6135a00852be01fffffffff88d448faef207016f19f336cd626b4a0976c5f39692638e66a92eb2962336d3190000006b483045022100a41a1b5747332cf3818a0650315241e4c1e12d783ecb49c468c7c5723e4e8c4d022077ae2654aee4c7980094dee2698e13d2a4ed876d1f0bf0f7fe29c9d31e780744012103782ffe90d1a5ae5ca00d6220f637a92d5469a056975c564e6b6135a00852be01ffffffff0130e3d1dc010000001976a914b63ae9d84b07d5056fa844fa2b51874936a8743688ac000000000c000000395e48a86b3de4d2866747847fc3f1301ecacc07a32911ca528fc2b69c716ad702b11f65dec9c0c95f2765d0826fe6858c82bf15c55c245a1752ff8fae033c98f7580000006b483045022100dd762919699fdc7762b8f202ce4ae19fc89a42a2da810651c0f92cfd3f92d75b022000b4b2106a529680a8b4c9fac81c37c73a4659636e75d66fc179dc3d164ea5fb0121022893da76884fec39a9504392b11cbe3b9d3a92ea27f22d6e180ec47b30c972dcffffffffe7e737b15a372d1716501abc3657706b1ae826f2ac6aaf0813e60cd0e435d389060000006b483045022100e9a9c0c7e1264b47343e6069dc9acc6dd357d9efb4309e4e6e9e4d47b4943a94022009fc6c0f72943fe4c117c00c698737bbd3dde029da576ada1b5ac1d5383d2b5e0121039c9611203386903f4df474764cbfa3b880c46916a8828ef44abe124e32324603ffffffff018c08ec01000000001976a914bbfaa9428a091607f13f9e3087e25befae4df2d088ac000000000c000000395e48a86b3de4d2866747847fc3f1301ecacc07a32911ca528fc2b69c716ad7060da576c407ca82c79438af58bdb79c53e4ac4f4092e5a7e0c29a80f7cb14d03e210000006b483045022100d46e87b453946e4a87032f7deefc35333f3208a1fa9e3d62796fa303388a601802201c58a77b49f0ee5c4d874465cc7f469117b6c3e59f79eddcff227c63ddcb961001210347d4e9666842b83843367dae294ead89d7252a723467a7c7a48e45c4ccd2d77bffffffffec354ad56a355961169e0fa85a048adb89a9cf2eee421521fe6acba6858fc6572f0000006b4830450221009a26aa35de2188b6513c9368db5d0e905d13bfea41ef13d9be9838707872be99022044a8c81eb5304026ef1d42f815add1c91d0a28b8873ffa141ff0d2baa8cb259f01210347d4e9666842b83Line truncated
+}

tests/blocks/bitcoingold_mainnet_499912.json

@@ -0,0 +1,15 @@
+{
+  "hash": "000000000dae8d873f300e1798e5c30353c831d7bdc048e559889f015dec4c25",
+  "size": 1889,
+  "height": 499912,
+  "merkleroot": "e29a320d6e2a00b49002acc6149f04d2aa90b4e37f80840f4ea26fb3b4949559",
+  "tx": [
+    "885c342d554e2bd9ba0b3b1b17da4d04fe88957e9d06ca922a26fb67c9ad3b91",
+    "a4c7785c6b5b6a1125af20e0bd01824919ed67730442ded5bc5ff5c1cb4f4dcd"
+  ],
+  "time": 1510594713,
+  "nonce": "01000000000000000000000003cfdc2609e4ff77000000000000000077ffe40a",
+  "bits": "1c14f7c1",
+  "previousblockhash": "00000000066db5fd7315c9dd8a260c1c38876e8f4dfa83904f4a682d5532f2ac",
+  "block": "00000020acf232552d684a4f9083fa4d8f6e87381c0c268addc91573fdb56d0600000000599594b4b36fa24e0f84807fe3b490aad2049f14c6ac0290b4002a6e0d329ae2c8a007000000000000000000000000000000000000000000000000000000000099d8095ac1f7141c0ae4ff77000000000000000077ffe40926dccf03000000000000000000000001fd40050124e0c3ae16f421a630c3d7f8762a596d96be0e9b114209262ae670a593b2250650fb467a7b6b3687fa111b0b1cf9c48177375dc597f449f549eff7d985721ede1177aece8d7bf77ed2ea44730751397eaa135b19b746c94058ae27d7ffd3f1a1ce1c62b0c51bab5e20af95eb6b159ddba38de547b6639ed5a4e952fb421caddf9aa6a8a5f56bc43622a0692f6aaf8816cafa1dc379c05895ffdd39d33310a275ccde7fce7a42b20329a13bd492010af88c04a016707caa4c00543b780460c7f55b8a962b3543e58e843a418689911a6e902d069461885911a5704338389f46906ae0f25b652e2dd2d6002b11394ceb6cf70587fae4f5eb162fa52105291dd53121423b7cc53217ae213b173464be083420c6c9df0a8d22031c17d3ffd35c4ac94335deb140062c91313d0ee1d3e883210d115c0dac5a934d6d7f47e5cfaaa8d4da88de3665b75f2fba46d7e29f8a8a01c019f20b9ff8eddc5701e5baf114ab5b9ebc1d5842c4e72d809c16b9a535c5dd15fd39adc628de07d803c4b16de64c7fd8dd421639aa3708060b4977e4dc05615045cfebcec5f608b2c15258ca30ce5dfb4a3305f4e95efa20e35335e820f92f4445e8d4c7f1602d108ae0da6b0af1dcf5ab41c1b750df398c69175c7d18aea4aa23ee929bf98ef7c7e8c62826f3a51fd7475f882bc09960b7d544a2b6d8f576e2ea545a1b9c9a0321c56e749018eb6e299df303f0d5db9db47ea5d51b58d72330d89a06f974e2417da13af914cfff768d1222a6725ec4982bd215d30079ed7fa28ade793b8a24f534f647e2b0c3e34219a6e6da2c6787331c66230ce2a0d3244c753cfb8890f51f66915d5d66d6d19f0f8f0f92eec9b1a5704292941146d476ae7bff4f6c2f268ae8fe97720ac2b288646e56b6a2cbd87e2ee43e6e8dde25a36a6bd0e618c3b6520822435d1d669f015a3e7b3f4b54036459831d793c230de7f157335f212fad0de34af75bab87e578a0b86aaf0953de7ac001cacb022b52f58329cef3eb96d383596dbc2ecb4c11b211cad30aade6a5acb2368c469dc515eb78574c04793774940a54e0a60711af10911682817cb7219e203e1a0e288c5652ac12c34a935c137a0dc4b1e73e0c47ef9d619599f76b3983a84eee8609a808ef20b315063ec39a5b16ade5ff348c04b1b9a2ad3f97e5da0170955f1fcc9dcb44d52408585bf721e760bfcb58277d2c289027ec0d58d9f32846a9593e16449bba8d06358e4d0cc455e535f722cb8b54e251e3b33f8b3a3623d5228ba2861fba56392ca26679b6620a9a3929144befe1e12df8f7c55d4385a2720caa6aec74f7e557b9a5441a5ff0cf761cba6385febf0f5f3b7ed44b18086bfea7a68e8de3711396b523966a05705fa9a32826c184315e0c7d30dce2c5a2d990ddf1a357d9f802b6748ae42c01fb8068f5361f4525be7f4f39f5790bd818672188319e4d01314a7a47e9ccb56c553c4107f9b21f0603f2bc8e9d50d4b8fa40b95891319141119c27cf79d4df7f0abc11219993c2daaf785ac3ec11f55298bcbd49fbea56691dc34fbe9713261e943824b5e714eae775e7a4133402224c7145f097108c2c18c2230efd4eeecf93bb53b03f5337f600181f606a408c76e08aa49eb5ee3505fd22ee1d16310ad7a0b504fcd2891f4d677f9af4667f6ae938cdb6dbcdbbd409f4c7efcfea2327a2c5218a114da119a0c1908f8e09caea4d9749bdecc88724552076ea9a2815df5ad60ddfeed42404e12d56eae28ebfcbcd46edaafc6f7e0f4fbffc6da0864b65dac66e7175abc1a2fa318a00182e8f7817537a2dde35460013d34a46c044fae6a114af14b4594e59734d4663d6f7ef503a9721b915b83ba1c6ec7e9d956bb40775873f32753564922fb6d50201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5b03c8a0070499d8095a1000000000000000000000000000000000403762383934396261636638653838666265363233663738383831363937316361616237623063613362373264643363393834386366343863623266316464336400000000016d8e814a000000001976a9143fd78e0eb6a079b9c48fec1501853da057f0f56d88ac0000000002000000019d26e7633ec88160ff5130da5c8f82cd9e7eef72acff2dcd3ca9e7ff102a5a54030000006a4730440220265e70787ceb0baa7685c8090bea7c1d02a2ed1f454328f4fa02bf954ad9e54c02202ae340ed2e8abea8f29854058c7509ce9b2cb3298d575e10d2d5ad969666416c4121033e2e2eaefbe75fdec3db57ea908cc44ac49289de1216eb11ce186d94849b8581feffffff02b160a32b000000001976a914b2743e71a671da1dfbc7430981d967a0b1d3c5ac88ac83093e00000000001976a91474c7cc96920ee3a78c982ddbb06a6e780c81669488acc7a00700"
+}

tests/blocks/bitcoingold_mainnet_80000.json

@@ -0,0 +1,15 @@
+{
+  "hash": "000000000043a8c0fd1d6f726790caa2a406010d19efd2780db27bdbbd93baf6",
+  "size": 373,
+  "height": 80000,
+  "merkleroot": "8fb300e3fdb6f30a4c67233b997f99fdd518b968b9a3fd65857bfe78b2600719",
+  "tx": [
+    "c06fbab289f723c6261d3030ddb6be121f7d2508d77862bb1e484f5cd7f92b25",
+    "5a4ebf66822b0b2d56bd9dc64ece0bc38ee7844a23ff1d7320a88c5fdb2ad3e2"
+  ],
+  "time": 1284613427,
+  "nonce": "00000000000000000000000000000000000000000000000000000000572fe301",
+  "bits": "1b5bede6",
+  "previousblockhash": "00000000001937917bd2caba204bb1aa530ec1de9d0f6736e5d85d96da9c8bba",
+  "block": "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f000000000000000000000000000000000000000000000000000000000000000033a5914ce6ed5b1b01e32f5700000000000000000000000000000000000000000000000000000000000201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000"
+}

tests/blocks/bitcoingreen_mainnet_10000.json

@@ -0,0 +1,14 @@
+{
+  "hash": "341452fa379115e62c7b27f9c86376ab8b7aabc4e39337765595edcb97d77930",
+  "size": 462,
+  "height": 10000,
+  "merkleroot": "fe847fdcf26d67b424959c6a9117bb54187eb661653bf2a208b5285eed239195",
+  "tx": [
+    "705e9918f8cb302b9c498396b89f61ce111830512a2809ef2febc315797f777f", "07d211091b51a1e2975ee07ba4933fa59f7aeda0818907bae3a7c497461b5f48"
+  ],
+  "time": 1517539297,
+  "nonce": 0,
+  "bits": "1b638c5b",
+  "previousblockhash": "7a9045c58a85b6d6a959721aef60ce5963897559a423e0e17ac1d34508a2ee53",
+  "block": "0400000053eea20845d3c17ae1e023a45975896359ce60ef1a7259a9d6b6858ac545907a959123ed5e28b508a2f23b6561b67e1854bb17916a9c9524b4676df2dc7f84fee1cf735a5b8c631b000000000201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff050210270101ffffffff0100000000000000000000000000010000000145ecb9083be84c146b1c96b88bf5157fe1771f7d6739bd137855c268e03da4c3020000006b4830450221009b4f8a1c8968c77192d38a538764a1fa5b33f9aef60e85c2157271759008033702202673c800cf8ce94259c0a06b9b093c9d1a716477ef9a1cb7c49c3c73ce7bbd880121023df508324112d8aa326d0539d741bf5f1f9dfc37d9f62b9b4574d76a1bd9600affffffff03000000000000000000005ed0b2000000002321023df508324112d8aa326d0539d741bf5f1f9dfc37d9f62b9b4574d76a1bd9600aac80e9fd97000000001976a914ee49dbdba7d30e659b93dde35d52c961ecbd186b88ac0000000046304402200320cb9881b72a30e81f5b2bfddc5bc404a3af8ef9220da8314896d7d025b5a0022070d9438034f84550b3a42fcaf86e7d186634956668b07702fa62d9db05b89366"
+}

tests/blocks/bitcoingreen_mainnet_285000.json

@@ -0,0 +1,14 @@
+{
+  "hash": "9b0276d83dff1883badf00a3d4cc2470cb8b5cb88fc8f7e506116d170ca13e25",
+  "size": 429,
+  "height": 285000,
+  "merkleroot": "ceac2a8b68bcfb5670c5acbeeca791ddfc4ed03b01beefb9e5e2d0f50462c8cb",
+  "tx": [
+    "5f381284dd7a496e1ebbabad5e870b28f108cc5bdc0ca8a93b54adb2611d16a3", "703b3f9a312c3eac763f7391905cfabba3faeea89f31b2952d287ab56c69965d"
+  ],
+  "time": 1534140284,
+  "nonce": 0,
+  "bits": "1b31a5bd",
+  "previousblockhash": "51670aa1f4b4564258c318edb8f2b1f4e7945154a138ac4516ee2d14e6939fed",
+  "block": "04000000ed9f93e6142dee1645ac38a1545194e7f4b1f2b8ed18c3584256b4f4a10a6751cbc86204f5d0e2e5b9efbe013bd04efcdd91a7ecbeacc57056fbbc688b2aacce7c1f715bbda5311b000000000201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff06034859040101ffffffff01000000000000000000000000000100000001871709454e15ee7ee23d602808a8b24a8f46ffdd883c788124827f78afd2292e0100000049483045022100db102575ef2f04151d302fd69ea1b5e5a45eb58e330d3fd0a1b42b3a688187dc02204f8380379afb4dbb05e6961add378a027dff5d0252d948b3bd2b89a9dc09856301ffffffff03000000000000000000cb8488150c000000232102d8e7ecc69382c4f201d76356074d79ef1378368cdffa01aff2fa21693e1a4933ac80f8a932000000001976a914f82aff4e5ed1a6b8d310145284371421e921c1a288ac000000004630440220646d7ab39036a29ae9e27805aa00d9dfcf1640601225cdd23daf9f8232f3f5eb02207a7bf880ffad041240bf01c304dc660796de4719a1827c9c2eb8bc84c3c01c29"
+}

tests/blocks/bitcoinplus_mainnet_749740.json

@@ -0,0 +1,15 @@
+{
+  "hash": "b51b081592349793bbf2d761828aabef40db8c7dd9451f0a0ea279faec1aecaf",
+  "size": 456,
+  "height": 749740,
+  "merkleroot": "cddd323b42891a0787389d40ea0971fbf4b0215ca40fc0229e94edf486eeb1dc",
+  "tx": [
+    "03079ad8d2c3e9db200582e7f3addec3f663c1388ea565482f6140ed58e83f1a", 
+    "6805f87aa7027c84db875d7147696dadd998cf261237bac4a3229e0d7d35c104"
+  ],
+  "time": 1544168075,
+  "nonce": 0,
+  "bits": "1e0cd5ed",
+  "previousblockhash": "d86428d967a901e169de1313276f2d463664179e5a7f303923811d1c4a177102",
+  "block": "030000200271174a1c1d812339307f5a9e176436462d6f271313de69e101a967d92864d8dcb1ee86f4ed949e22c00fa45c21b0f4fb7109ea409d3887071a89423b32ddcd8b220a5cedd50c1e0000000002010000008b220a5c010000000000000000000000000000000000000000000000000000000000000000ffffffff0503ac700b00ffffffff0200000000000000000000000000000000000000000000010000008b220a5c015186024f3de419c8846da76a254930f9b5d68246eb21b1aceca5ba889a190ba30100000049483045022100d81ae320b3f1a296cae778b342937412cc4689f375d5dfc594c1df7fac6aa046022015c71b090e85579f041ae31c6d1881959414231882648b0878a886ba8b8b2a5a01ffffffff03000000000000000000408491a400000000232102eeb314d0ae50152e8f416521b0f3184325f57d8c7932d6a91d186fa2820d3da7ac56aca7a400000000232102eeb314d0ae50152e8f416521b0f3184325f57d8c7932d6a91d186fa2820d3da7ac00000000473045022100c4a62b764a0984e6e2f909ca41b21f61b17f230d60f6c2ba69cb504300931bc202200aa57659cf8c54861002ca924b018385aa3a44c4ff0bc9207ddecc3b8086f76e"
+}

tests/blocks/bitcoinprivate_mainnet_1000.json

@@ -0,0 +1,24 @@
+{
+    "anchor": "55cf34dabe7c224490b1db308a297e00ce515c3fd6fdfddbb57e4b76e5b74969",
+    "bits": "1d497ffb",
+    "chainwork": "00000000000000000000000000000000000000000000000000000004fa21155b",
+    "confirmations": 679241,
+    "difficulty": 7133.163866900871,
+    "hash": "000000311c66325218753433b49ec25150dc220af017fe1198e33f5ad4f71a51",
+    "height": 1000,
+    "merkleroot": "0f69dc56186005ca2f720cafa6253131ae0c9eeac779dbb59ba56633ceb852eb",
+    "nextblockhash": "0000002b7c4ecaf3bb80ddda4212f635a7bd2958e391c150b66c715b892cbd58",
+    "nonce": "0000000000000000000000001113ee60a7ddff0f00000000000000000fffdda8",
+    "previousblockhash": "0000003230d263146297e1d18985e942615e967df4d7d2acf8c352da42858c90",
+    "size": 4959,
+    "solution": "001198f8b4c65f187a45a760e7cddf31ed8cd8537c00927d59a095c14b75416048c7e3cf4554fc6bb533152d729e89693e6bdda6ea0c24db4906f5507dd6b24c166c53959e6bd977e95540ceac4cbd9156ba116e0abbf21e8def663fa44ae1e68a686312a00bdf4f4311f24216ae1be6119c70a38e9231723926c50f909816c7d9367c4dd1a9cb6de49078ffcf7333107e8dbe68b10dcfcae290a779ca0756eac667cecf3f5ae306019bcfd276045f0aa7ac71778ef4e548933497caac1bf8142dc259550b056bc23ecb138498b7db523eb50b04dec186a4df996e6601084c9252b5d51ffa1f0733680b86aa0e8a110b31037375b1713721041eef7b08e809696517785727dbd46f9edf758ec1fadc7f490b5f69294dc78413198e52d605d30cee94d6fa23a80a16220331da87d5892a825174ef14fdfdc7fb0636409172057a5099b2a4591700c948cd934f3dfc103f00569e40e0e8223d617d43c3edec994eb1d435b7a41a9bab47d5296bd5c031e21a2162ad410ba11c49df021076899a15d7a53e43205aea1f75e1104e539adc08b39958560b9762a963a75084bfa57a587758fe190168ff1ff067e86361b5923685cfc5d68976b8a18e0c14c41ad0cd523ed1e71231d7c25cfabe77beffa60a2cfaba6fe57785c9990263f29badc2144e34ab853014bb23250dac929badc3cd787203761f221dee11026d9f115bc3be4842242416e5f9da5d55b6bf1c8e3ae779fe485599f5fb1807cfa954fa7e321bd3475b0691fc87ac52b7a89935f494446963db03659847582bb5e582946970b7ca4812eb589ff14cd9814de3d504fb3f9b61e75ee9654a37812e67748684ac97a17249b8ab0668e063793f598931e7e4d76abe273ce6c32af91e61e2d3f90790260671f3fc9412c6d5f9cf144d8f454edbd836ed73d524e77b548ba72cd19bfd700029f27dff06c6595ecc24ab12d86f914aa82d455d02741f3a9f91cf99ac0540e952c28a82ebdcfdaf12404a8320bef4a397ec96253ddf71d6a9df75f6762f54dd6fc627d690936e933a89b8672bceeef09784f90727de604c9f9d3322b39491b73c13154e7bb784061e5a98f6045a4df32da461faf515a0a538f65b73da19246b1afce6ea85a6c851a3e4bcf1f0e7281e2cc7195275aa1346cd2ffa8fe70776d72a067922fcbb5f064a1e1107db226f6262810bae7479de2d699908d61d791b9ad390635abbff56cd1966263281cdbdc566070228c91dcbddd1f01720d2bf0c356523db1669800cde1ea888ad6c4d70161104f1ecf0928e80d9a46c06cf60dc36cd326d6ee3013c3ba0525631f21d57351c55618e8ee46af79a172412d2f72d859d3f78951c26ded3e80710033bb8b8e635f0dccb49c7a32ea15734980cccf51133a7b4dd9602187984c273211c55900275b5c4e23a32cfe733a17d5b1df70ca28e6a631324fd346be08c6aa86b56c8b5f8508042a4f4ff705009abb551061e9177d037f0fbf7992b6ee769b890771ea024dee88ee20fddb0d4703ca7c1b2bb5576bf790a355e455cc9445a52a9c7faf55cb2138bb8df9448108c989046c4387fb42966635aec4ecf2efadc18470ffaca18f64a1bffe75842defa5255f35fe49e9a62162ee5e42b4a5277971f318458de110e08389232bd0414c9134ec3b7684da382db4897305500d1cadd2e049c1d86c8a2973dc5f5057498ad3e718615ed861314525bfaf5f965c9ee97219e29c00aa6271fb408dd47ae368c121a4cc14b89b95a5452575e787479cfb617030a676c0c1ba7fcded85259e6aed29e5c1e153818043ad2740eefbb0baed39fdca511ad6a6932395427843f72e2a3303d6a8804878b25d5a9a07d9d61344bdee2f755d62763e14e65081de9add165d4beaaf0",
+    "time": 1478513393,
+    "tx": [
+      "6f54c6101380ae8a481da9545d585c687e420de6e68f90ca4d26ef168eaff951",
+      "b5356193fa8d48abe0bb3a767c1e116900ecbffefe0920348273a8dae96ad8a3",
+      "5553aafeb26e154bfd80063235f36a212c82497680e87db23a2228275113eb9f"
+    ],
+    "version": 4,
+    "block": "04000000908c8542da52c3f8acd2d7f47d965e6142e98589d1e197621463d23032000000eb52b8ce3366a59bb5db79c7ea9e0cae313125a6af0c722fca05601856dc690f0000000000000000000000000000000000000000000000000000000000000000f1522058fb7f491da8ddff0f00000000000000000fffdda760ee1311000000000000000000000000fd4005001198f8b4c65f187a45a760e7cddf31ed8cd8537c00927d59a095c14b75416048c7e3cf4554fc6bb533152d729e89693e6bdda6ea0c24db4906f5507dd6b24c166c53959e6bd977e95540ceac4cbd9156ba116e0abbf21e8def663fa44ae1e68a686312a00bdf4f4311f24216ae1be6119c70a38e9231723926c50f909816c7d9367c4dd1a9cb6de49078ffcf7333107e8dbe68b10dcfcae290a779ca0756eac667cecf3f5ae306019bcfd276045f0aa7ac71778ef4e548933497caac1bf8142dc259550b056bc23ecb138498b7db523eb50b04dec186a4df996e6601084c9252b5d51ffa1f0733680b86aa0e8a110b31037375b1713721041eef7b08e809696517785727dbd46f9edf758ec1fadc7f490b5f69294dc78413198e52d605d30cee94d6fa23a80a16220331da87d5892a825174ef14fdfdc7fb0636409172057a5099b2a4591700c948cd934f3dfc103f00569e40e0e8223d617d43c3edec994eb1d435b7a41a9bab47d5296bd5c031e21a2162ad410ba11c49df021076899a15d7a53e43205aea1f75e1104e539adc08b39958560b9762a963a75084bfa57a587758fe190168ff1ff067e86361b5923685cfc5d68976b8a18e0c14c41ad0cd523ed1e71231d7c25cfabe77beffa60a2cfaba6fe57785c9990263f29badc2144e34ab853014bb23250dac929badc3cd787203761f221dee11026d9f115bc3be4842242416e5f9da5d55b6bf1c8e3ae779fe485599f5fb1807cfa954fa7e321bd3475b0691fc87ac52b7a89935f494446963db03659847582bb5e582946970b7ca4812eb589ff14cd9814de3d504fb3f9b61e75ee9654a37812e67748684ac97a17249b8ab0668e063793f598931e7e4d76abe273ce6c32af91e61e2d3f90790260671f3fc9412c6d5f9cf144d8f454edbd836ed73d524e77b548ba72cd19bfd700029f27dff06c6595ecc24ab12d86f914aa82d455d02741f3a9f91cf99ac0540e952c28a82ebdcfdaf12404a8320bef4a397ec96253ddf71d6a9df75f6762f54dd6fc627d690936e933a89b8672bceeef09784f90727de604c9f9d3322b39491b73c13154e7bb784061e5a98f6045a4df32da461faf515a0a538f65b73da19246b1afce6ea85a6c851a3e4bcf1f0e7281e2cc7195275aa1346cd2ffa8fe70776d72a067922fcbb5f064a1e1107db226f6262810bae7479de2d699908d61d791b9ad390635abbff56cd1966263281cdbdc566070228c91dcbddd1f01720d2bf0c356523db1669800cde1ea888ad6c4d70161104f1ecf0928e80d9a46c06cf60dc36cd326d6ee3013c3ba0525631f21d57351c55618e8ee46af79a172412d2f72d859d3f78951c26ded3e80710033bb8b8e635f0dccb49c7a32ea15734980cccf51133a7b4dd9602187984c273211c55900275b5c4e23a32cfe733a17d5b1df70ca28e6a631324fd346be08c6aa86b56c8b5f8508042a4f4ff705009abb551061e9177d037f0fbf7992b6ee769b890771ea024dee88ee20fddb0d4703ca7c1b2bb5576bf790a355e455cc9445a52a9c7faf55cb2138bb8df9448108c989046c4387fb42966635aec4ecf2efadc18470ffaca18f64a1bffe75842defa5255f35fe49e9a62162ee5e42b4a5277971f318458de110e08389232bd0414c9134ec3b7684da382db4897305500d1cadd2e049c1d86c8a2973dc5f5057498ad3e718615ed861314525bfaf5f965c9ee97219e29c00aa6271fb408dd47ae368c121a4cc14b89b95a5452575e787479cfb617030a676c0c1ba7fcded85259e6aed29e5c1e153818043ad2740eefbb0baed39fdca511ad6a6932395427843f72e2a3303d6a8804878b25d5a9a07d9d61344bdee2f755d62763e14e65081de9add165d4beaaf00301000000010000000000000000000000000000000000000000000000000000000000000000ffffffff2702e80304f152205810000000000000000000000000000000000d2f6e6f64655374726174756d2f0000000001807c814a000000002321024c171766283550d8c0af521ca567402c1197296c36f911d4bdb6e62dc8da583fac0000000001000000022629a8d1bb75bc3d70536d59d7a06c519fcd3dbca89e4ace4ffd9e6c00f807a2170000006b483045022100994174fd2339e73d6d7d3a43a69e6038bb055b6b01bb24dfa56c767695bfc5d702203221c9bdfa66ea800abeeef8063e8920d6cac4f66e5f1562c127dd456cfe7cf9012102d9f2b80ecaec33606cf5fa4738df935ca2fcbb4649de8f6ee36d3aac33c7a72afeffffff40166e25466bfa447fd0c279635885370b9e3ad98ea8dc865ea655c0fd2144e5200000006b483045022100db37b31344a86ed440dd5a92ef0aa37960da8e1a3790d8c22a3299a40fceec3b0220456c43eb77fab217be1a1b43b3a9ff244f4e2cd51cbdf0badc0480727c0b14a70121022018a5510d0f224ab01e7b0aa174f51737aab7303b082985a143ce002f5d1030feffffff209f721800000000001976a914567f61e0dd2ff69bc019f744b7f780ec81e87c8a88ac9d3a1600000000001976a9143fefaa5b1f358830143845a730d3f6e31e122fe188ac10281d06000000001976a91499bf5ca7c8cec2bd7a5e8f1a2e9f5015376be5dd88ac68903000000000001976a91482a233b9d53698d3d64cec8a86db6bd84bdc8fd988ac3c351200000000001976a914ec5d438d8fe079f82601880762fa4789f66d421988acf2be4d00000000001976a91450baa06a424859340b8abe72b8ba66325fff910488acf6ab5f00000000001976a914a9794119fb8ec322fff43cbe956d49dd62fd0c3688ace60d6000000000001976a914a8137e1113cf068d93521abc2c1f4f368df22ba788ac264c1600000000001976a914811e4d7b6785a5434ab9e447d0232066a17b3fd088acba843700000000001976a91445a2aa563f8c13cb929f992cdc82198169a3e43c88ac99301d00000000001976a914eb8e449a604423512216a1418147093c7655548e88accaf76600000000001976a9144a1f93cedfa0663217ccfb52511af8ff81eb16fc88ac03bb1000000000001976a91412e7ac3ebc41db3bebab8cef7036d875be4f208988ac458f0a03000000001976a9143d12750c723c0fade83f6297c88445129e53eaec88ac6aa95e00000000001976a914d9f502757e52240f255114ed3069d5f669ea49af88acea744c06000000001976a9143ebec661b0ff61ecc7103b7acfa8cc5aa2aae55988ac0d2ea100000000001976a9146cdf98cb44818cc72d2a0c5a4Line truncated
+  }
+