Initial commit: snapshot of spesmilo/electrumx 2.0.0

Unmodified source tree of upstream commit 280cb339aa158e099eb936033eecd15719770651
("Merge pull request #389 from SomberNight/202607_release_2_0",
https://github.com/spesmilo/electrumx, release 2.0.0, 3 July 2026).

All Palladium-specific changes are in the commits that follow.

src/electrumx/lib/enum.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2016, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''An enum-like type with reverse lookup.
+
+Source: Python Cookbook, http://code.activestate.com/recipes/67107/
+'''
+
+
+class EnumError(Exception):
+    pass
+
+
+class Enumeration:
+
+    def __init__(self, name, enumList):
+        self.__doc__ = name
+
+        lookup = {}
+        reverseLookup = {}
+        i = 0
+        uniqueNames = set()
+        uniqueValues = set()
+        for x in enumList:
+            if isinstance(x, tuple):
+                x, i = x
+            if not isinstance(x, str):
+                raise EnumError("enum name {} not a string".format(x))
+            if not isinstance(i, int):
+                raise EnumError("enum value {} not an integer".format(i))
+            if x in uniqueNames:
+                raise EnumError("enum name {} not unique".format(x))
+            if i in uniqueValues:
+                raise EnumError("enum value {} not unique".format(x))
+            uniqueNames.add(x)
+            uniqueValues.add(i)
+            lookup[x] = i
+            reverseLookup[i] = x
+            i = i + 1
+        self.lookup = lookup
+        self.reverseLookup = reverseLookup
+
+    def __getattr__(self, attr):
+        result = self.lookup.get(attr)
+        if result is None:
+            raise AttributeError('enumeration has no member {}'.format(attr))
+        return result
+
+    def whatis(self, value):
+        return self.reverseLookup[value]

src/electrumx/lib/env_base.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2017, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Class for server environment configuration and defaults.'''
+
+
+from os import environ
+
+from electrumx.lib.util import class_logger
+
+
+class EnvBase:
+    '''Wraps environment configuration.'''
+
+    class Error(Exception):
+        pass
+
+    def __init__(self):
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.allow_root = self.boolean('ALLOW_ROOT', False)
+        self.loop_policy = self.event_loop_policy()
+
+    @classmethod
+    def default(cls, envvar, default):
+        return environ.get(envvar, default)
+
+    @classmethod
+    def boolean(cls, envvar, default):
+        default = 'Yes' if default else ''
+        return bool(cls.default(envvar, default).strip())
+
+    @classmethod
+    def required(cls, envvar):
+        value = environ.get(envvar)
+        if value is None:
+            raise cls.Error(f'required envvar {envvar} not set')
+        return value
+
+    @classmethod
+    def integer(cls, envvar, default):
+        value = environ.get(envvar)
+        if value is None:
+            return default
+        try:
+            return int(value)
+        except Exception:
+            raise cls.Error(f'cannot convert envvar {envvar} value {value} to '
+                            f'an integer')
+
+    @classmethod
+    def custom(cls, envvar, default, parse):
+        value = environ.get(envvar)
+        if value is None:
+            return default
+        try:
+            return parse(value)
+        except Exception as e:
+            raise cls.Error(
+                f'cannot parse envvar {envvar} value {value}'
+            ) from e
+
+    @classmethod
+    def obsolete(cls, envvars):
+        bad = [envvar for envvar in envvars if environ.get(envvar)]
+        if bad:
+            raise cls.Error(f'remove obsolete environment variables {bad}')
+
+    def event_loop_policy(self):
+        policy = self.default('EVENT_LOOP_POLICY', None)
+        if policy is None:
+            return None
+        if policy == 'uvloop':
+            import uvloop
+            return uvloop.EventLoopPolicy()
+        raise self.Error(f'unknown event loop policy "{policy}"')

src/electrumx/lib/hash.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Cryptograph hash functions and related classes.'''
+
+
+import hashlib
+import hmac
+
+from electrumx.lib.util import bytes_to_int, int_to_bytes, hex_to_bytes
+
+_sha256 = hashlib.sha256
+_new_hash = hashlib.new
+_hmac_digest = hmac.digest
+HASHX_LEN = 11
+
+
+def sha256(x: bytes) -> bytes:
+    '''Simple wrapper of hashlib sha256.'''
+    return _sha256(x).digest()
+
+
+def double_sha256(x: bytes) -> bytes:
+    '''SHA-256 of SHA-256, as used extensively in bitcoin.'''
+    return sha256(sha256(x))
+
+
+def hash_to_hex_str(x: bytes) -> str:
+    '''Convert a big-endian binary hash to displayed hex string. (rev-to-hum)
+
+    Display form of a binary hash is reversed and converted to hex.
+    '''
+    return bytes(reversed(x)).hex()
+
+
+def hex_str_to_hash(x: str) -> bytes:
+    '''Convert a displayed hex string to a binary hash. (hum-to-rev)'''
+    return bytes(reversed(hex_to_bytes(x)))
+
+
+class Base58Error(Exception):
+    '''Exception used for Base58 errors.'''
+
+
+class Base58:
+    '''Class providing base 58 functionality.'''
+
+    chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+    assert len(chars) == 58
+    cmap = {c: n for n, c in enumerate(chars)}
+
+    @staticmethod
+    def char_value(c):
+        val = Base58.cmap.get(c)
+        if val is None:
+            raise Base58Error(f'invalid base 58 character "{c}"')
+        return val
+
+    @staticmethod
+    def decode(txt):
+        """Decodes txt into a big-endian bytearray."""
+        if not isinstance(txt, str):
+            raise TypeError('a string is required')
+
+        if not txt:
+            raise Base58Error('string cannot be empty')
+
+        value = 0
+        for c in txt:
+            value = value * 58 + Base58.char_value(c)
+
+        result = int_to_bytes(value)
+
+        # Prepend leading zero bytes if necessary
+        count = 0
+        for c in txt:
+            if c != '1':
+                break
+            count += 1
+        if count:
+            result = bytes(count) + result
+
+        return result
+
+    @staticmethod
+    def encode(be_bytes):
+        """Converts a big-endian bytearray into a base58 string."""
+        value = bytes_to_int(be_bytes)
+
+        txt = ''
+        while value:
+            value, mod = divmod(value, 58)
+            txt += Base58.chars[mod]
+
+        for byte in be_bytes:
+            if byte != 0:
+                break
+            txt += '1'
+
+        return txt[::-1]
+
+    @staticmethod
+    def decode_check(txt, *, hash_fn=double_sha256):
+        '''Decodes a Base58Check-encoded string to a payload.  The version
+        prefixes it.'''
+        be_bytes = Base58.decode(txt)
+        result, check = be_bytes[:-4], be_bytes[-4:]
+        if check != hash_fn(result)[:4]:
+            raise Base58Error(f'invalid base 58 checksum for {txt}')
+        return result
+
+    @staticmethod
+    def encode_check(payload, *, hash_fn=double_sha256):
+        """Encodes a payload bytearray (which includes the version byte(s))
+        into a Base58Check string."""
+        be_bytes = payload + hash_fn(payload)[:4]
+        return Base58.encode(be_bytes)

src/electrumx/lib/lrucache.py

@@ -0,0 +1,190 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2014-2022 Thomas Kemmer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# -----
+#
+# This is a stripped down LRU-cache from the "cachetools" library.
+# https://github.com/tkem/cachetools/blob/d991ac71b4eb6394be5ec572b835434081393215/src/cachetools/__init__.py
+
+import collections
+import collections.abc
+from typing import TypeVar, Dict
+
+
+class _DefaultSize:
+
+    __slots__ = ()
+
+    def __getitem__(self, _):
+        return 1
+
+    def __setitem__(self, _, value):
+        assert value == 1
+
+    def pop(self, _):
+        return 1
+
+
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
+
+
+class Cache(collections.abc.MutableMapping[_KT, _VT]):
+    """Mutable mapping to serve as a simple cache or cache base class."""
+
+    __marker = object()
+
+    __size = _DefaultSize()
+
+    def __init__(self, maxsize: int, getsizeof=None):
+        if getsizeof:
+            self.getsizeof = getsizeof
+        if self.getsizeof is not Cache.getsizeof:
+            self.__size = dict()
+        self.__data = dict()  # type: Dict[_KT, _VT]
+        self.__currsize = 0
+        self.__maxsize = maxsize
+        # these can be used externally to keep track of cache hit statistics:
+        # (proper internal bookkeeping w/o double-counting seems either expensive or complicated)
+        self.num_lookups = 0
+        self.num_hits = 0
+
+    def __repr__(self):
+        return "%s(%s, maxsize=%r, currsize=%r)" % (
+            self.__class__.__name__,
+            repr(self.__data),
+            self.__maxsize,
+            self.__currsize,
+        )
+
+    def __getitem__(self, key: _KT) -> _VT:
+        try:
+            return self.__data[key]
+        except KeyError:
+            return self.__missing__(key)
+
+    def __setitem__(self, key: _KT, value: _VT) -> None:
+        maxsize = self.__maxsize
+        size = self.getsizeof(value)
+        if size > maxsize:
+            raise ValueError("value too large")
+        if key not in self.__data or self.__size[key] < size:
+            while self.__currsize + size > maxsize:
+                self.popitem()
+        if key in self.__data:
+            diffsize = size - self.__size[key]
+        else:
+            diffsize = size
+        self.__data[key] = value
+        self.__size[key] = size
+        self.__currsize += diffsize
+
+    def __delitem__(self, key: _KT) -> None:
+        size = self.__size.pop(key)
+        del self.__data[key]
+        self.__currsize -= size
+
+    def __contains__(self, key: _KT) -> bool:
+        return key in self.__data
+
+    def __missing__(self, key: _KT):
+        raise KeyError(key)
+
+    def __iter__(self):
+        return iter(self.__data)
+
+    def __len__(self):
+        return len(self.__data)
+
+    def get(self, key: _KT, default: _VT = None) -> _VT | None:
+        if key in self:
+            return self[key]
+        else:
+            return default
+
+    def pop(self, key: _KT, default=__marker) -> _VT:
+        if key in self:
+            value = self[key]
+            del self[key]
+        elif default is self.__marker:
+            raise KeyError(key)
+        else:
+            value = default
+        return value
+
+    def setdefault(self, key: _KT, default: _VT = None) -> _VT | None:
+        if key in self:
+            value = self[key]
+        else:
+            self[key] = value = default
+        return value
+
+    @property
+    def maxsize(self) -> int:
+        """The maximum size of the cache."""
+        return self.__maxsize
+
+    @property
+    def currsize(self) -> int:
+        """The current size of the cache."""
+        return self.__currsize
+
+    @staticmethod
+    def getsizeof(value) -> int:
+        """Return the size of a cache element's value."""
+        return 1
+
+
+class LRUCache(Cache[_KT, _VT]):
+    """Least Recently Used (LRU) cache implementation."""
+
+    def __init__(self, maxsize: int, getsizeof=None):
+        Cache.__init__(self, maxsize, getsizeof)
+        self.__order = collections.OrderedDict()
+
+    def __getitem__(self, key: _KT, cache_getitem=Cache.__getitem__) -> _VT | None:
+        value = cache_getitem(self, key)
+        if key in self:  # __missing__ may not store item
+            self.__update(key)
+        return value
+
+    def __setitem__(self, key: _KT, value, cache_setitem=Cache.__setitem__) -> None:
+        cache_setitem(self, key, value)
+        self.__update(key)
+
+    def __delitem__(self, key: _KT, cache_delitem=Cache.__delitem__) -> None:
+        cache_delitem(self, key)
+        del self.__order[key]
+
+    def popitem(self) -> tuple[_KT, _VT]:
+        """Remove and return the `(key, value)` pair least recently used."""
+        try:
+            key = next(iter(self.__order))
+        except StopIteration:
+            raise KeyError("%s is empty" % type(self).__name__) from None
+        else:
+            return (key, self.pop(key))
+
+    def __update(self, key: _KT) -> None:
+        try:
+            self.__order.move_to_end(key)
+        except KeyError:
+            self.__order[key] = None

src/electrumx/lib/merkle.py

@@ -0,0 +1,266 @@
+# Copyright (c) 2018, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+'''Merkle trees, branches, proofs and roots.'''
+
+from math import ceil, log
+from typing import Sequence, Optional, Callable, Tuple, Iterable, List, Awaitable
+
+from aiorpcx import Event, run_in_thread
+
+from electrumx.lib.hash import double_sha256
+from electrumx.lib.util import class_logger
+
+
+class Merkle:
+    '''Perform merkle tree calculations on binary hashes using a given hash
+    function.
+
+    If the hash count is not even, the final hash is repeated when
+    calculating the next merkle layer up the tree.
+    '''
+
+    def __init__(self, hash_func: Callable[[bytes], bytes] = double_sha256):
+        self.hash_func = hash_func
+
+    def tree_depth(self, hash_count: int) -> int:
+        return self.branch_length(hash_count) + 1
+
+    def branch_length(self, hash_count: int) -> int:
+        '''Return the length of a merkle branch given the number of hashes.'''
+        if not isinstance(hash_count, int):
+            raise TypeError('hash_count must be an integer')
+        if hash_count < 1:
+            raise ValueError('hash_count must be at least 1')
+        return ceil(log(hash_count, 2))
+
+    def branch_and_root(
+            self, hashes: Iterable[bytes], index: int, length: Optional[int] = None,
+    ) -> Tuple[List[bytes], bytes]:
+        '''Return a (merkle branch, merkle_root) pair given hashes, and the
+        index of one of those hashes.
+        '''
+        hashes = list(hashes)
+        if not isinstance(index, int):
+            raise TypeError('index must be an integer')
+        # This also asserts hashes is not empty
+        if not 0 <= index < len(hashes):
+            raise ValueError('index out of range')
+        natural_length = self.branch_length(len(hashes))
+        if length is None:
+            length = natural_length
+        else:
+            if not isinstance(length, int):
+                raise TypeError('length must be an integer')
+            if length < natural_length:
+                raise ValueError('length out of range')
+
+        hash_func = self.hash_func
+        branch = []
+        for _ in range(length):
+            if len(hashes) & 1:
+                hashes.append(hashes[-1])
+            branch.append(hashes[index ^ 1])
+            index >>= 1
+            hashes = [hash_func(hashes[n] + hashes[n + 1])
+                      for n in range(0, len(hashes), 2)]
+
+        return branch, hashes[0]
+
+    def root(self, hashes: Sequence[bytes], length: Optional[int] = None) -> bytes:
+        '''Return the merkle root of a non-empty iterable of binary hashes.'''
+        _branch, root = self.branch_and_root(hashes, 0, length)
+        return root
+
+    def root_from_proof(self, hash: bytes, branch: Iterable[bytes], index: int) -> bytes:
+        '''Return the merkle root given a hash, a merkle branch to it, and
+        its index in the hashes array.
+
+        branch is an iterable sorted deepest to shallowest.  If the
+        returned root is the expected value then the merkle proof is
+        verified.
+
+        The caller should have confirmed the length of the branch with
+        branch_length().  Unfortunately this is not easily done for
+        bitcoin transactions as the number of transactions in a block
+        is unknown to an SPV client.
+        '''
+        hash_func = self.hash_func
+        for elt in branch:
+            if index & 1:
+                hash = hash_func(elt + hash)
+            else:
+                hash = hash_func(hash + elt)
+            index >>= 1
+        if index:
+            raise ValueError('index out of range for branch')
+        return hash
+
+    def level(self, hashes: Sequence[bytes], depth_higher: int) -> List[bytes]:
+        '''Return a level of the merkle tree of hashes the given depth
+        higher than the bottom row of the original tree.'''
+        size = 1 << depth_higher
+        root = self.root
+        return [root(hashes[n: n + size], depth_higher)
+                for n in range(0, len(hashes), size)]
+
+    def branch_and_root_from_level(
+            self, level: List[bytes], leaf_hashes: Sequence[bytes], index: int, depth_higher: int,
+    ) -> Tuple[List[bytes], bytes]:
+        '''Return a (merkle branch, merkle_root) pair when a merkle-tree has a
+        level cached.
+
+        To maximally reduce the amount of data hashed in computing a
+        merkle branch, cache a tree of depth N at level N // 2.
+
+        level is a list of hashes in the middle of the tree (returned
+        by level())
+
+        leaf_hashes are the leaves needed to calculate a partial branch
+        up to level.
+
+        depth_higher is how much higher level is than the leaves of the tree
+
+        index is the index in the full list of hashes of the hash whose
+        merkle branch we want.
+        '''
+        if not isinstance(level, list):
+            raise TypeError("level must be a list")
+        if not isinstance(leaf_hashes, list):
+            raise TypeError("leaf_hashes must be a list")
+        leaf_index = (index >> depth_higher) << depth_higher
+        leaf_branch, leaf_root = self.branch_and_root(
+            leaf_hashes, index - leaf_index, depth_higher)
+        index >>= depth_higher
+        level_branch, root = self.branch_and_root(level, index)
+        # Check last so that we know index is in-range
+        if leaf_root != level[index]:
+            raise ValueError('leaf hashes inconsistent with level')
+        return leaf_branch + level_branch, root
+
+
+class MerkleCache:
+    '''A cache to calculate merkle branches efficiently.'''
+
+    def __init__(
+            self,
+            merkle: 'Merkle',
+            source_func: Callable[[int, int], Awaitable[Sequence[bytes]]],
+    ):
+        '''Initialise a cache hashes taken from source_func:
+
+           async def source_func(index, count):
+              ...
+        '''
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        self.merkle = merkle
+        self.source_func = source_func
+        self.length = 0  # type: int
+        self.level = []  # type: List[bytes]
+        self.depth_higher = 0  # type: int
+        self.initialized = Event()
+
+    def _segment_length(self) -> int:
+        return 1 << self.depth_higher
+
+    def _leaf_start(self, index: int) -> int:
+        '''Given a level's depth higher and a hash index, return the leaf
+        index and leaf hash count needed to calculate a merkle branch.
+        '''
+        depth_higher = self.depth_higher
+        return (index >> depth_higher) << depth_higher
+
+    def _level(self, hashes: Sequence[bytes]) -> List[bytes]:
+        return self.merkle.level(hashes, self.depth_higher)
+
+    async def _extend_to(self, length: int) -> None:
+        '''Extend the length of the cache if necessary.'''
+        if length <= self.length:
+            return
+        # Start from the beginning of any final partial segment.
+        # Retain the value of depth_higher; in practice this is fine
+        start = self._leaf_start(self.length)
+        hashes = await self.source_func(start, length - start)
+        self.level[start >> self.depth_higher:] = self._level(hashes)
+        self.length = length
+
+    async def _level_for(self, length: int) -> List[bytes]:
+        '''Return a (level_length, final_hash) pair for a truncation
+        of the hashes to the given length.'''
+        if length == self.length:
+            return self.level
+        level = self.level[:length >> self.depth_higher]
+        leaf_start = self._leaf_start(length)
+        count = min(self._segment_length(), length - leaf_start)
+        hashes = await self.source_func(leaf_start, count)
+        level += self._level(hashes)
+        return level
+
+    async def initialize(self, length: int) -> None:
+        '''Call to initialize the cache to a source of given length.'''
+        self.length = length
+        self.depth_higher = self.merkle.tree_depth(length) // 2
+        hashes = await self.source_func(0, length)
+        self.level = await run_in_thread(self._level, hashes)
+        self.initialized.set()
+
+    def truncate(self, length: int) -> None:
+        '''Truncate the cache so it covers no more than length underlying
+        hashes.'''
+        if not isinstance(length, int):
+            raise TypeError('length must be an integer')
+        if length <= 0:
+            raise ValueError('length must be positive')
+        if length >= self.length:
+            return
+        length = self._leaf_start(length)
+        self.length = length
+        self.level[length >> self.depth_higher:] = []
+
+    async def branch_and_root(self, length: int, index: int) -> Tuple[List[bytes], bytes]:
+        '''Return a merkle branch and root.  Length is the number of
+        hashes used to calculate the merkle root, index is the position
+        of the hash to calculate the branch of.
+
+        index must be less than length, which must be at least 1.'''
+        if not isinstance(length, int):
+            raise TypeError('length must be an integer')
+        if not isinstance(index, int):
+            raise TypeError('index must be an integer')
+        if length <= 0:
+            raise ValueError('length must be positive')
+        if index >= length:
+            raise ValueError('index must be less than length')
+        await self.initialized.wait()
+        await self._extend_to(length)
+        leaf_start = self._leaf_start(index)
+        count = min(self._segment_length(), length - leaf_start)
+        leaf_hashes = await self.source_func(leaf_start, count)
+        if length < self._segment_length():
+            return self.merkle.branch_and_root(leaf_hashes, index)
+        level = await self._level_for(length)
+        return self.merkle.branch_and_root_from_level(
+            level, leaf_hashes, index, self.depth_higher)

src/electrumx/lib/peer.py

@@ -0,0 +1,337 @@
+# Copyright (c) 2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Representation of a peer server.'''
+
+from functools import cached_property
+from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network
+from socket import AF_INET, AF_INET6
+from typing import Any
+
+from aiorpcx import is_valid_hostname
+from electrumx.lib.util import protocol_tuple, version_string
+
+
+class Peer:
+
+    # Protocol version
+    ATTRS = ('host', 'features',
+             # metadata
+             'source', 'ip_addr',
+             'last_good', 'last_try', 'try_count')
+    FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max',
+                'ssl_port', 'tcp_port')
+    # This should be set by the application
+    DEFAULT_PORTS = {}
+
+    def __init__(self, host, features, source='unknown', ip_addr=None,
+                 last_good=0, last_try=0, try_count=0):
+        '''Create a peer given a host name (or IP address as a string),
+        a dictionary of features, and a record of the source.'''
+        assert isinstance(host, str)
+        assert isinstance(features, dict)
+        assert host in features.get('hosts', {})
+        self.host = host
+        self.features = features.copy()
+        # Canonicalize / clean-up
+        for feature in self.FEATURES:
+            self.features[feature] = getattr(self, feature)
+        # Metadata
+        self.source = source
+        self.ip_addr = ip_addr
+        # last_good represents the last connection that was
+        # successful *and* successfully verified, at which point
+        # try_count is set to 0.  Failure to connect or failure to
+        # verify increment the try_count.
+        self.last_good = last_good
+        self.last_try = last_try
+        self.try_count = try_count
+        # Transient, non-persisted metadata
+        self.bad = False
+        self.other_port_pairs = set()
+
+    @classmethod
+    def peers_from_features(cls, features: dict[str, Any] | Any, source):
+        peers = []
+        if isinstance(features, dict):
+            hosts = features.get('hosts')
+            if isinstance(hosts, dict):
+                peers = [Peer(host, features, source=source)
+                         for host in hosts if isinstance(host, str)]
+        return peers
+
+    @classmethod
+    def deserialize(cls, item):
+        '''Deserialize from a dictionary.'''
+        return cls(**item)
+
+    def matches(self, peers):
+        '''Return peers whose host matches our hostname or IP address.
+        Additionally include all peers whose IP address matches our
+        hostname if that is an IP address.
+        '''
+        candidates = (self.host.lower(), self.ip_addr)
+        return [peer for peer in peers
+                if peer.host.lower() in candidates
+                or peer.ip_addr == self.host]
+
+    def __str__(self):
+        return self.host
+
+    def update_features(self, features):
+        '''Update features in-place.'''
+        try:
+            tmp = Peer(self.host, features)
+        except AssertionError:
+            pass
+        else:
+            self.update_features_from_peer(tmp)
+
+    def update_features_from_peer(self, peer):
+        if peer != self:
+            self.features = peer.features
+            for feature in self.FEATURES:
+                setattr(self, feature, getattr(peer, feature))
+
+    def connection_tuples(self):
+        '''Return a list of (kind, port, family) tuples to try when making a
+        connection.
+        '''
+        # Use a list not a set - it's important to try the registered
+        # ports first.
+        pairs = [('SSL', self.ssl_port), ('TCP', self.tcp_port)]
+        while self.other_port_pairs:
+            pairs.append(self.other_port_pairs.pop())
+        if isinstance(self.ip_address, IPv4Address):
+            families = (AF_INET,)
+        elif isinstance(self.ip_address, IPv6Address):
+            families = (AF_INET6,)
+        else:
+            families = (AF_INET, AF_INET6)
+        return [(kind, port, family)
+                for kind, port in pairs if port
+                for family in families]
+
+    def mark_bad(self):
+        '''Mark as bad to avoid reconnects but also to remember for a
+        while.'''
+        self.bad = True
+
+    def check_ports(self, other):
+        '''Remember differing ports in case server operator changed them
+        or removed one.'''
+        if other.ssl_port != self.ssl_port:
+            self.other_port_pairs.add(('SSL', other.ssl_port))
+        if other.tcp_port != self.tcp_port:
+            self.other_port_pairs.add(('TCP', other.tcp_port))
+        return bool(self.other_port_pairs)
+
+    @cached_property
+    def is_tor(self):
+        return self.host.endswith('.onion')
+
+    @cached_property
+    def is_valid(self):
+        ip = self.ip_address
+        if ip:
+            return ((ip.is_global or ip.is_private)
+                    and not (ip.is_multicast or ip.is_unspecified))
+        return is_valid_hostname(self.host)
+
+    @cached_property
+    def is_public(self):
+        ip = self.ip_address
+        if ip:
+            return self.is_valid and not ip.is_private
+        else:
+            return self.is_valid and self.host != 'localhost'
+
+    @cached_property
+    def ip_address(self):
+        '''The host as a python ip_address object, or None.'''
+        try:
+            return ip_address(self.host)
+        except ValueError:
+            return None
+
+    def bucket_for_internal_purposes(self):
+        '''Used for keeping the internal peer list manageable in size.
+        Restrictions are loose.
+        '''
+        if self.is_tor:
+            return 'onion'
+        if not self.ip_addr:
+            return ''
+        ip_addr = ip_address(self.ip_addr)
+        if ip_addr.version == 4:
+            return str(ip_addr)
+        elif ip_addr.version == 6:
+            slash64 = IPv6Network(self.ip_addr).supernet(prefixlen_diff=128-64)
+            return str(slash64)
+        return ''
+
+    def bucket_for_external_interface(self):
+        '''Used when responding to RPC queries to return a distributed list
+        of peers. Restrictions are stricter than internal bucketing.
+        '''
+        if self.is_tor:
+            return 'onion'
+        if not self.ip_addr:
+            return ''
+        ip_addr = ip_address(self.ip_addr)
+        if ip_addr.version == 4:
+            slash16 = IPv4Network(self.ip_addr).supernet(prefixlen_diff=32-16)
+            return str(slash16)
+        elif ip_addr.version == 6:
+            slash56 = IPv6Network(self.ip_addr).supernet(prefixlen_diff=128-56)
+            return str(slash56)
+        return ''
+
+    def serialize(self):
+        '''Serialize to a dictionary.'''
+        return {attr: getattr(self, attr) for attr in self.ATTRS}
+
+    def _port(self, key):
+        hosts = self.features.get('hosts')
+        if isinstance(hosts, dict):
+            host = hosts.get(self.host)
+            port = self._integer(key, host)
+            if port and 0 < port < 65536:
+                return port
+        return None
+
+    def _integer(self, key, d=None):
+        d = d or self.features
+        result = d.get(key) if isinstance(d, dict) else None
+        if isinstance(result, str):
+            try:
+                result = int(result)
+            except ValueError:
+                pass
+        return result if isinstance(result, int) else None
+
+    def _string(self, key):
+        result = self.features.get(key)
+        return result if isinstance(result, str) else None
+
+    @cached_property
+    def genesis_hash(self):
+        '''Returns the network genesis block hash as a string if known, otherwise None.'''
+        return self._string('genesis_hash')
+
+    @cached_property
+    def ssl_port(self):
+        '''Returns None if no SSL port, otherwise the port as an integer.'''
+        return self._port('ssl_port')
+
+    @cached_property
+    def tcp_port(self):
+        '''Returns None if no TCP port, otherwise the port as an integer.'''
+        return self._port('tcp_port')
+
+    @cached_property
+    def server_version(self):
+        '''Returns the server version as a string if known, otherwise None.'''
+        return self._string('server_version')
+
+    @cached_property
+    def pruning(self):
+        '''Returns the pruning level as an integer.  None indicates no
+        pruning.'''
+        pruning = self._integer('pruning')
+        if pruning and pruning > 0:
+            return pruning
+        return None
+
+    def _protocol_version_string(self, key):
+        version_str = self.features.get(key)
+        ptuple = protocol_tuple(version_str)
+        return version_string(ptuple)
+
+    @cached_property
+    def protocol_min(self):
+        '''Minimum protocol version as a string, e.g., 1.0'''
+        return self._protocol_version_string('protocol_min')
+
+    @cached_property
+    def protocol_max(self):
+        '''Maximum protocol version as a string, e.g., 1.1'''
+        return self._protocol_version_string('protocol_max')
+
+    def to_tuple(self):
+        '''The tuple ((ip, host, details) expected in response
+        to a peers subscription.'''
+        details = self.real_name().split()[1:]
+        return (self.ip_addr or self.host, self.host, details)
+
+    def real_name(self):
+        '''Real name of this peer as used on IRC.'''
+        def port_text(letter, port):
+            if port == self.DEFAULT_PORTS.get(letter):
+                return letter
+            else:
+                return letter + str(port)
+
+        parts = [self.host, 'v' + self.protocol_max]
+        if self.pruning:
+            parts.append(f'p{self.pruning:d}')
+        for letter, port in (('s', self.ssl_port), ('t', self.tcp_port)):
+            if port:
+                parts.append(port_text(letter, port))
+        return ' '.join(parts)
+
+    @classmethod
+    def from_real_name(cls, real_name, source):
+        '''Real name is a real name as on IRC, such as
+
+            "erbium1.sytes.net v1.0 s t"
+
+        Returns an instance of this Peer class.
+        '''
+        host = 'nohost'
+        features = {}
+        ports = {}
+        for n, part in enumerate(real_name.split()):
+            if n == 0:
+                host = part
+                continue
+            if part[0] in ('s', 't'):
+                if len(part) == 1:
+                    port = cls.DEFAULT_PORTS[part[0]]
+                else:
+                    port = part[1:]
+                if part[0] == 's':
+                    ports['ssl_port'] = port
+                else:
+                    ports['tcp_port'] = port
+            elif part[0] == 'v':
+                features['protocol_max'] = features['protocol_min'] = part[1:]
+            elif part[0] == 'p':
+                features['pruning'] = part[1:]
+
+        features.update(ports)
+        features['hosts'] = {host: ports}
+
+        return cls(host, features, source)

src/electrumx/lib/script.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+'''Script-related classes and functions.'''
+from electrumx.lib.enum import Enumeration
+from electrumx.lib.util import unpack_le_uint16_from, unpack_le_uint32_from, \
+    pack_le_uint16, pack_le_uint32
+
+
+class ScriptError(Exception):
+    '''Exception used for script errors.'''
+
+
+OpCodes = Enumeration("Opcodes", [
+    ("OP_0", 0), ("OP_PUSHDATA1", 76),
+    "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE",
+    "OP_RESERVED",
+    "OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7", "OP_8",
+    "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
+    "OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF",
+    "OP_ELSE", "OP_ENDIF", "OP_VERIFY", "OP_RETURN",
+    "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP",
+    "OP_2OVER", "OP_2ROT", "OP_2SWAP", "OP_IFDUP", "OP_DEPTH", "OP_DROP",
+    "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL", "OP_ROT",
+    "OP_SWAP", "OP_TUCK",
+    "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE",
+    "OP_INVERT", "OP_AND", "OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY",
+    "OP_RESERVED1", "OP_RESERVED2",
+    "OP_1ADD", "OP_1SUB", "OP_2MUL", "OP_2DIV", "OP_NEGATE", "OP_ABS",
+    "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL", "OP_DIV", "OP_MOD",
+    "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR", "OP_NUMEQUAL",
+    "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN", "OP_GREATERTHAN",
+    "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
+    "OP_WITHIN",
+    "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160", "OP_HASH256",
+    "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
+    "OP_CHECKMULTISIGVERIFY",
+    "OP_NOP1",
+    "OP_CHECKLOCKTIMEVERIFY", "OP_CHECKSEQUENCEVERIFY"
+])
+
+
+# Paranoia to make it hard to create bad scripts
+assert OpCodes.OP_DUP == 0x76
+assert OpCodes.OP_HASH160 == 0xa9
+assert OpCodes.OP_EQUAL == 0x87
+assert OpCodes.OP_EQUALVERIFY == 0x88
+assert OpCodes.OP_CHECKSIG == 0xac
+assert OpCodes.OP_CHECKMULTISIG == 0xae
+
+
+def is_unspendable_legacy(script):
+    # OP_FALSE OP_RETURN or OP_RETURN
+    return script[:2] == b'\x00\x6a' or (script and script[0] == 0x6a)
+
+
+def is_unspendable_genesis(script):
+    # OP_FALSE OP_RETURN
+    return script[:2] == b'\x00\x6a'
+
+
+def _match_ops(ops, pattern):
+    if len(ops) != len(pattern):
+        return False
+    for op, pop in zip(ops, pattern):
+        if pop != op:
+            # -1 means 'data push', whose op is an (op, data) tuple
+            if pop == -1 and isinstance(op, tuple):
+                continue
+            return False
+
+    return True
+
+
+class ScriptPubKey:
+    '''A class for handling a tx output script that gives conditions
+    necessary for spending.
+    '''
+
+    TO_ADDRESS_OPS = (OpCodes.OP_DUP, OpCodes.OP_HASH160, -1,
+                      OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG)
+    TO_P2SH_OPS = (OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL)
+    TO_PUBKEY_OPS = (-1, OpCodes.OP_CHECKSIG)
+
+    @classmethod
+    def P2SH_script(cls, hash160):
+        return (bytes((OpCodes.OP_HASH160,))
+                + Script.push_data(hash160)
+                + bytes((OpCodes.OP_EQUAL,)))
+
+    @classmethod
+    def P2PKH_script(cls, hash160):
+        return (bytes((OpCodes.OP_DUP, OpCodes.OP_HASH160))
+                + Script.push_data(hash160)
+                + bytes((OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG)))
+
+
+class Script:
+
+    @classmethod
+    def get_ops(cls, script):
+        ops = []
+
+        # The unpacks or script[n] below throw on truncated scripts
+        try:
+            n = 0
+            while n < len(script):
+                op = script[n]
+                n += 1
+
+                if op <= OpCodes.OP_PUSHDATA4:
+                    # Raw bytes follow
+                    if op < OpCodes.OP_PUSHDATA1:
+                        dlen = op
+                    elif op == OpCodes.OP_PUSHDATA1:
+                        dlen = script[n]
+                        n += 1
+                    elif op == OpCodes.OP_PUSHDATA2:
+                        dlen, = unpack_le_uint16_from(script[n: n + 2])
+                        n += 2
+                    else:
+                        dlen, = unpack_le_uint32_from(script[n: n + 4])
+                        n += 4
+                    if n + dlen > len(script):
+                        raise IndexError
+                    op = (op, script[n:n + dlen])
+                    n += dlen
+
+                ops.append(op)
+        except Exception:
+            # Truncated script; e.g. tx_hash
+            # ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
+            raise ScriptError('truncated script')
+
+        return ops
+
+    @classmethod
+    def push_data(cls, data):
+        '''Returns the opcodes to push the data on the stack.'''
+        assert isinstance(data, (bytes, bytearray))
+
+        n = len(data)
+        if n < OpCodes.OP_PUSHDATA1:
+            return bytes((n,)) + data
+        if n < 256:
+            return bytes((OpCodes.OP_PUSHDATA1, n)) + data
+        if n < 65536:
+            return bytes((OpCodes.OP_PUSHDATA2,)) + pack_le_uint16(n) + data
+        return bytes((OpCodes.OP_PUSHDATA4,)) + pack_le_uint32(n) + data
+
+    @classmethod
+    def opcode_name(cls, opcode):
+        if OpCodes.OP_0 < opcode < OpCodes.OP_PUSHDATA1:
+            return f'OP_{opcode:d}'
+        try:
+            return OpCodes.whatis(opcode)
+        except KeyError:
+            return f'OP_UNKNOWN:{opcode:d}'
+
+    @classmethod
+    def dump(cls, script):
+        opcodes, datas = cls.get_ops(script)
+        for opcode, data in zip(opcodes, datas):
+            name = cls.opcode_name(opcode)
+            if data is None:
+                print(name)
+            else:
+                print(f'{name} {data.hex()} ({len(data):d} bytes)')

src/electrumx/lib/server_base.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2017, Neil Booth
+#
+# All rights reserved.
+#
+# See the file "LICENCE" for information about the copyright
+# and warranty status of this software.
+
+'''Base class of servers'''
+
+import asyncio
+import os
+import platform
+import re
+import signal
+import sys
+import time
+from contextlib import suppress
+from functools import partial
+from typing import TYPE_CHECKING
+
+from aiorpcx import spawn
+
+from electrumx import PYTHON_MIN_VERSION
+from electrumx.lib.util import class_logger
+
+if TYPE_CHECKING:
+    from electrumx.server.env import Env
+
+
+class ServerBase:
+    '''Base class server implementation.
+
+    Derived classes are expected to:
+
+    - set SUPPRESS_MESSAGE_REGEX as appropriate
+    - implement the serve() coroutine, called from the run() method.
+      Upon return the event loop runs until the shutdown signal is received.
+    '''
+    SUPPRESS_MESSAGE_REGEX = re.compile('SSL handshake|Fatal read error on|'
+                                        'SSL error in data received|'
+                                        'socket.send() raised exception')
+    SUPPRESS_TASK_REGEX = re.compile('accept_connection2')
+
+    def __init__(self, env: 'Env'):
+        '''Save the environment, perform basic sanity checks, and set the
+        event loop policy.
+        '''
+        # First asyncio operation must be to set the event loop policy
+        # as this replaces the event loop
+        asyncio.set_event_loop_policy(env.loop_policy)
+
+        self.logger = class_logger(__name__, self.__class__.__name__)
+        version_str = ' '.join(sys.version.splitlines())
+        self.logger.info(f'Python version: {version_str}')
+        self.env = env
+        self.start_time = 0
+
+        # Sanity checks
+        if sys.version_info < PYTHON_MIN_VERSION:
+            mvs = '.'.join(str(part) for part in PYTHON_MIN_VERSION)
+            raise RuntimeError(f'Python version >= {mvs} is required')
+
+        if platform.system() == 'Windows':
+            pass
+        elif os.geteuid() == 0 and not env.allow_root:
+            raise RuntimeError('RUNNING AS ROOT IS STRONGLY DISCOURAGED!\n'
+                               'You shoud create an unprivileged user account '
+                               'and use that.\n'
+                               'To continue as root anyway, restart with '
+                               'environment variable ALLOW_ROOT non-empty')
+
+    async def serve(self, shutdown_event: asyncio.Event):
+        '''Override to provide the main server functionality.
+        Run as a task that will be cancelled to request shutdown.
+
+        Setting the event also shuts down the server.
+        '''
+
+    def on_exception(self, loop, context):
+        '''Suppress spurious messages it appears we cannot control.'''
+        message = context.get('message')
+        if message and self.SUPPRESS_MESSAGE_REGEX.match(message):
+            return
+        if self.SUPPRESS_TASK_REGEX.match(repr(context.get('task'))):
+            return
+        loop.default_exception_handler(context)
+
+    async def run(self):
+        '''Run the server application:
+
+        - record start time
+        - install SIGINT and SIGTERM handlers to trigger shutdown_event
+        - set loop's exception handler to suppress unwanted messages
+        - run the event loop until serve() completes
+        '''
+        def on_signal(signame):
+            shutdown_event.set()
+            self.logger.warning(f'received {signame} signal, initiating shutdown')
+
+        async def serve():
+            try:
+                await self.serve(shutdown_event)
+            finally:
+                shutdown_event.set()
+
+        self.start_time = time.time()
+        loop = asyncio.get_event_loop()
+        shutdown_event = asyncio.Event()
+
+        if platform.system() != 'Windows':
+            # No signals on Windows
+            for signame in ('SIGINT', 'SIGTERM'):
+                loop.add_signal_handler(getattr(signal, signame),
+                                        partial(on_signal, signame))
+        loop.set_exception_handler(self.on_exception)
+
+        # Start serving and wait for shutdown, log receipt of the event
+        server_task = await spawn(serve, daemon=True)
+        try:
+            await shutdown_event.wait()
+        except KeyboardInterrupt:
+            self.logger.warning('received keyboard interrupt, initiating shutdown')
+
+        self.logger.info('shutting down')
+
+        server_task.cancel()
+        try:
+            with suppress(asyncio.CancelledError):
+                await server_task
+        finally:
+            self.logger.info('shutdown complete')

src/electrumx/lib/text.py

@@ -0,0 +1,86 @@
+import time
+
+import electrumx.lib.util as util
+
+
+def sessions_lines(data):
+    '''A generator returning lines for a list of sessions.
+
+    data is the return value of rpc_sessions().'''
+    fmt = ('{:<6} {:<5} {:>17} {:>5} '
+           '{:>7} {:>7} {:>5} {:>5} {:>7} '
+           '{:>7} {:>7} {:>7} {:>7} {:>9} {:>21}')
+    yield fmt.format('ID', 'Flags', 'Client', 'Proto',
+                     'Cost', 'XCost', 'Reqs', 'Txs', 'Subs',
+                     'Recv', 'Recv KB', 'Sent', 'Sent KB', 'Time', 'Peer')
+    for (id_, flags, peer, client, proto, cost, extra_cost, reqs, txs_sent, subs,
+         recv_count, recv_size, send_count, send_size, conn_time) in data:
+        yield fmt.format(id_, flags, client, proto,
+                         f'{int(cost):,d}',
+                         f'{int(extra_cost):,d}',
+                         f'{reqs:,d}',
+                         f'{txs_sent:,d}',
+                         f'{subs:,d}',
+                         f'{recv_count:,d}',
+                         f'{recv_size // 1024:,d}',
+                         f'{send_count:,d}',
+                         f'{send_size // 1024:,d}',
+                         util.formatted_time(conn_time, sep=''), peer)
+
+
+def groups_lines(data):
+    '''A generator returning lines for a list of groups.
+
+    data is the return value of rpc_groups().'''
+
+    fmt = ('{:<14} {:>9} {:>8} {:>8} {:>6} {:>6} {:>8}'
+           '{:>7} {:>9} {:>7} {:>9}')
+    yield fmt.format('Name', 'Sessions', 'Cost', 'Retained', 'Reqs', 'Txs', 'Subs',
+                     'Recv', 'Recv KB', 'Sent', 'Sent KB')
+    for (name, session_count, cost, retained_cost, reqs, txs_sent, subs,
+         recv_count, recv_size, send_count, send_size) in data:
+        yield fmt.format(name,
+                         f'{session_count:,d}',
+                         f'{int(cost):,d}',
+                         f'{int(retained_cost):,d}',
+                         f'{reqs:,d}',
+                         f'{txs_sent:,d}',
+                         f'{subs:,d}',
+                         f'{recv_count:,d}',
+                         f'{recv_size // 1024:,d}',
+                         f'{send_count:,d}',
+                         f'{send_size // 1024:,d}')
+
+
+def peers_lines(data):
+    '''A generator returning lines for a list of peers.
+
+    data is the return value of rpc_peers().'''
+    def time_fmt(t):
+        if not t:
+            return 'Never'
+        return util.formatted_time(now - t)
+
+    now = time.time()
+    fmt = ('{:<62} {:<6} {:>5} {:>5} {:<17} {:>4} '
+           '{:>4} {:>8} {:>11} {:>11} {:>5} {:>20} {:<15}')
+    yield fmt.format('Host', 'Status', 'TCP', 'SSL', 'Server', 'Min',
+                     'Max', 'Pruning', 'Last Good', 'Last Try',
+                     'Tries', 'Source', 'IP Address')
+    for item in data:
+        features = item['features']
+        hostname = item['host']
+        host = features['hosts'][hostname]
+        yield fmt.format(hostname[:62],
+                         item['status'],
+                         host.get('tcp_port') or '',
+                         host.get('ssl_port') or '',
+                         features['server_version'] or 'unknown',
+                         features['protocol_min'],
+                         features['protocol_max'],
+                         features['pruning'] or '',
+                         time_fmt(item['last_good']),
+                         time_fmt(item['last_try']),
+                         item['try_count'],
+                         item['source'][:20],
+                         item['ip_addr'] or '')

src/electrumx/lib/tx_axe.py

@@ -0,0 +1,489 @@
+# Copyright (c) 2016-2018, Neil Booth
+# Copyright (c) 2018, the ElectrumX authors
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Deserializer for AXE DIP2 special transaction types'''
+
+from dataclasses import dataclass
+from typing import Any
+
+from electrumx.lib.tx import Deserializer, Tx
+from electrumx.lib.util import (pack_le_uint16, pack_le_int32, pack_le_uint32,
+                                pack_le_int64, pack_varint, pack_varbytes,
+                                pack_be_uint16)
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002.md
+@dataclass(kw_only=True, slots=True)
+class AxeTx(Tx):
+    '''Class representing a Axe transaction'''
+    tx_type: int
+    extra_payload: Any
+
+    def serialize(self):
+        nLocktime = pack_le_uint32(self.locktime)
+        txins = (pack_varint(len(self.inputs)) +
+                 b''.join(tx_in.serialize() for tx_in in self.inputs))
+        txouts = (pack_varint(len(self.outputs)) +
+                  b''.join(tx_out.serialize() for tx_out in self.outputs))
+
+        if self.tx_type:
+            uVersion = pack_le_uint16(self.version)
+            uTxType = pack_le_uint16(self.tx_type)
+            vExtra = self._serialize_extra_payload()
+            return uVersion + uTxType + txins + txouts + nLocktime + vExtra
+        else:
+            nVersion = pack_le_int32(self.version)
+            return nVersion + txins + txouts + nLocktime
+
+    def _serialize_extra_payload(self):
+        extra = self.extra_payload
+        spec_tx_class = DeserializerAxe.SPEC_TX_HANDLERS.get(self.tx_type)
+        if not spec_tx_class:
+            assert isinstance(extra, (bytes, bytearray))
+            return pack_varbytes(extra)
+
+        if not isinstance(extra, spec_tx_class):
+            raise ValueError('Axe tx_type does not conform with extra'
+                             ' payload class: %s, %s' % (self.tx_type, extra))
+        return pack_varbytes(extra.serialize())
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002-special-transactions.md
+@dataclass(kw_only=True, slots=True)
+class AxeProRegTx:
+    '''Class representing DIP3 ProRegTx'''
+    version: int
+    type: int
+    mode: int
+    collateralOutpoint: 'TxOutPoint'
+    ipAddress: bytes
+    port: int
+    KeyIdOwner: bytes
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    operatorReward: int
+    scriptPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.ipAddress) == 16
+                and len(self.KeyIdOwner) == 20
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        return (
+            pack_le_uint16(self.version) +              # version
+            pack_le_uint16(self.type) +                 # type
+            pack_le_uint16(self.mode) +                 # mode
+            self.collateralOutpoint.serialize() +       # collateralOutpoint
+            self.ipAddress +                            # ipAddress
+            pack_be_uint16(self.port) +                 # port
+            self.KeyIdOwner +                           # KeyIdOwner
+            self.PubKeyOperator +                       # PubKeyOperator
+            self.KeyIdVoting +                          # KeyIdVoting
+            pack_le_uint16(self.operatorReward) +       # operatorReward
+            pack_varbytes(self.scriptPayout) +          # scriptPayout
+            self.inputsHash +                           # inputsHash
+            pack_varbytes(self.payloadSig)              # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProRegTx(
+            version=deser._read_le_uint16(),
+            type=deser._read_le_uint16(),
+            mode=deser._read_le_uint16(),
+            collateralOutpoint=deser._read_outpoint(),
+            ipAddress=deser._read_nbytes(16),
+            port=deser._read_be_uint16(),
+            KeyIdOwner=deser._read_nbytes(20),
+            PubKeyOperator=deser._read_nbytes(48),
+            KeyIdVoting=deser._read_nbytes(20),
+            operatorReward=deser._read_le_uint16(),
+            scriptPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_varbytes(),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeProUpServTx:
+    '''Class representing DIP3 ProUpServTx'''
+    version: int
+    proTxHash: bytes
+    ipAddress: bytes
+    port: int
+    scriptOperatorPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.ipAddress) == 16
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            self.ipAddress +                            # ipAddress
+            pack_be_uint16(self.port) +                 # port
+            pack_varbytes(self.scriptOperatorPayout) +  # scriptOperatorPayout
+            self.inputsHash +                           # inputsHash
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProUpServTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            ipAddress=deser._read_nbytes(16),
+            port=deser._read_be_uint16(),
+            scriptOperatorPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeProUpRegTx:
+    '''Class representing DIP3 ProUpRegTx'''
+    version: int
+    proTxHash: bytes
+    mode: int
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    scriptPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.mode) +                 # mode
+            self.PubKeyOperator +                       # PubKeyOperator
+            self.KeyIdVoting +                          # KeyIdVoting
+            pack_varbytes(self.scriptPayout) +          # scriptPayout
+            self.inputsHash +                           # inputsHash
+            pack_varbytes(self.payloadSig)              # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProUpRegTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            mode=deser._read_le_uint16(),
+            PubKeyOperator=deser._read_nbytes(48),
+            KeyIdVoting=deser._read_nbytes(20),
+            scriptPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_varbytes(),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeProUpRevTx:
+    '''Class representing DIP3 ProUpRevTx'''
+    version: int
+    proTxHash: bytes
+    reason: int
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.reason) +               # reason
+            self.inputsHash +                           # inputsHash
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeProUpRevTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            reason=deser._read_le_uint16(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeCbTx:
+    '''Class representing DIP4 coinbase special tx'''
+    version: int
+    height: int
+    merkleRootMNList: bytes
+    merkleRootQuorums: bytes
+
+    def serialize(self):
+        assert len(self.merkleRootMNList) == 32
+        res = (
+            pack_le_uint16(self.version) +              # version
+            pack_le_uint32(self.height) +               # height
+            self.merkleRootMNList                       # merkleRootMNList
+        )
+        if self.version > 1:
+            assert len(self.merkleRootQuorums) == 32
+            res += self.merkleRootQuorums               # merkleRootQuorums
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()
+        height = deser._read_le_uint32()
+        merkleRootMNList = deser._read_nbytes(32)
+        merkleRootQuorums = b''
+        if version > 1:
+            merkleRootQuorums = deser._read_nbytes(32)
+        return AxeCbTx(
+            version=version,
+            height=height,
+            merkleRootMNList=merkleRootMNList,
+            merkleRootQuorums=merkleRootQuorums,
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxRegister:
+    '''Class representing DIP5 SubTxRegister'''
+    version: int
+    userName: bytes
+    pubKey: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.pubKey) == 48
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            pack_varbytes(self.userName) +              # userName
+            self.pubKey +                               # pubKey
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxRegister(
+            version=deser._read_le_uint16(),
+            userName=deser._read_varbytes(),
+            pubKey=deser._read_nbytes(48),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxTopup:
+    '''Class representing DIP5 SubTxTopup'''
+    version: int
+    regTxHash: bytes
+
+    def serialize(self):
+        assert len(self.regTxHash) == 32
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.regTxHash                              # regTxHash
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxTopup(
+            version=deser._read_le_uint16(),
+            regTxHash=deser._read_nbytes(32),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxResetKey:
+    '''Class representing DIP5 SubTxResetKey'''
+    version: int
+    regTxHash: bytes
+    hashPrevSubTx: bytes
+    creditFee: int
+    newPubKey: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.regTxHash) == 32
+                and len(self.hashPrevSubTx) == 32
+                and len(self.newPubKey) == 48
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.regTxHash +                            # regTxHash
+            self.hashPrevSubTx +                        # hashPrevSubTx
+            pack_le_int64(self.creditFee) +             # creditFee
+            self.newPubKey +                            # newPubKey
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxResetKey(
+            version=deser._read_le_uint16(),
+            regTxHash=deser._read_nbytes(32),
+            hashPrevSubTx=deser._read_nbytes(32),
+            creditFee=deser._read_le_int64(),
+            newPubKey=deser._read_nbytes(48),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class AxeSubTxCloseAccount:
+    '''Class representing DIP5 SubTxCloseAccount'''
+    version: int
+    regTxHash: bytes
+    hashPrevSubTx: bytes
+    creditFee: int
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.regTxHash) == 32
+                and len(self.hashPrevSubTx) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.regTxHash +                            # regTxHash
+            self.hashPrevSubTx +                        # hashPrevSubTx
+            pack_le_int64(self.creditFee) +             # creditFee
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return AxeSubTxCloseAccount(
+            version=deser._read_le_uint16(),
+            regTxHash=deser._read_nbytes(32),
+            hashPrevSubTx=deser._read_nbytes(32),
+            creditFee=deser._read_le_int64(),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+# https://dash-docs.github.io/en/developer-reference#outpoint
+@dataclass(kw_only=True, slots=True)
+class TxOutPoint:
+    '''Class representing tx output outpoint'''
+    hash: bytes
+    index: int
+
+    def serialize(self):
+        assert len(self.hash) == 32
+        return (
+            self.hash +                                 # hash
+            pack_le_uint32(self.index)                  # index
+        )
+
+    @classmethod
+    def read_outpoint(cls, deser):
+        return TxOutPoint(
+            hash=deser._read_nbytes(32),
+            index=deser._read_le_uint32(),
+        )
+
+
+class DeserializerAxe(Deserializer):
+    '''Deserializer for AXE DIP2 special tx types'''
+    # Supported Spec Tx types and corresponding classes mapping
+    PRO_REG_TX = 1
+    PRO_UP_SERV_TX = 2
+    PRO_UP_REG_TX = 3
+    PRO_UP_REV_TX = 4
+    CB_TX = 5
+    SUB_TX_REGISTER = 8
+    SUB_TX_TOPUP = 9
+    SUB_TX_RESET_KEY = 10
+    SUB_TX_CLOSE_ACCOUNT = 11
+
+    SPEC_TX_HANDLERS = {
+        PRO_REG_TX: AxeProRegTx,
+        PRO_UP_SERV_TX: AxeProUpServTx,
+        PRO_UP_REG_TX: AxeProUpRegTx,
+        PRO_UP_REV_TX: AxeProUpRevTx,
+        CB_TX: AxeCbTx,
+        SUB_TX_REGISTER: AxeSubTxRegister,
+        SUB_TX_TOPUP: AxeSubTxTopup,
+        SUB_TX_RESET_KEY: AxeSubTxResetKey,
+        SUB_TX_CLOSE_ACCOUNT: AxeSubTxCloseAccount,
+    }
+
+    def _read_outpoint(self):
+        return TxOutPoint.read_outpoint(self)
+
+    def read_tx(self):
+        start = self.cursor
+        header = self._read_le_uint32()
+        tx_type = header >> 16  # DIP2 tx type
+        if tx_type:
+            version = header & 0x0000ffff
+        else:
+            version = header
+
+        if tx_type and version < 3:
+            version = header
+            tx_type = 0
+
+        inputs = self._read_inputs()
+        outputs = self._read_outputs()
+        locktime = self._read_le_uint32()
+        if tx_type:
+            extra_payload_size = self._read_varint()
+            end = self.cursor + extra_payload_size
+            spec_tx_class = DeserializerAxe.SPEC_TX_HANDLERS.get(tx_type)
+            if spec_tx_class:
+                read_method = getattr(spec_tx_class, 'read_tx_extra', None)
+                extra_payload = read_method(self)
+                assert isinstance(extra_payload, spec_tx_class)
+            else:
+                extra_payload = self._read_nbytes(extra_payload_size)
+            assert self.cursor == end
+        else:
+            extra_payload = b''
+        txid = self.TX_HASH_FN(self.binary[start:self.cursor])
+        tx = AxeTx(
+            version=version,
+            inputs=inputs,
+            outputs=outputs,
+            locktime=locktime,
+            tx_type=tx_type,
+            extra_payload=extra_payload,
+            txid_rev=txid,
+            wtxid_rev=txid,
+        )
+        return tx

src/electrumx/lib/tx_dash.py

@@ -0,0 +1,442 @@
+# Copyright (c) 2016-2018, Neil Booth
+# Copyright (c) 2018, the ElectrumX authors
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''Deserializer for Dash DIP2 special transaction types'''
+
+from dataclasses import dataclass
+from typing import Any
+
+from electrumx.lib.tx import Deserializer, Tx
+from electrumx.lib.util import (pack_le_uint16, pack_le_int32, pack_le_uint32,
+                                pack_le_int64, pack_varint, pack_varbytes,
+                                pack_be_uint16)
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002.md
+@dataclass(kw_only=True, slots=True)
+class DashTx(Tx):
+    '''Class representing a Dash transaction'''
+    tx_type: int
+    extra_payload: Any
+
+    def serialize(self):
+        nLocktime = pack_le_uint32(self.locktime)
+        txins = (pack_varint(len(self.inputs)) +
+                 b''.join(tx_in.serialize() for tx_in in self.inputs))
+        txouts = (pack_varint(len(self.outputs)) +
+                  b''.join(tx_out.serialize() for tx_out in self.outputs))
+
+        if self.tx_type:
+            uVersion = pack_le_uint16(self.version)
+            uTxType = pack_le_uint16(self.tx_type)
+            vExtra = self._serialize_extra_payload()
+            return uVersion + uTxType + txins + txouts + nLocktime + vExtra
+        else:
+            nVersion = pack_le_int32(self.version)
+            return nVersion + txins + txouts + nLocktime
+
+    def _serialize_extra_payload(self):
+        extra = self.extra_payload
+        spec_tx_class = DeserializerDash.SPEC_TX_HANDLERS.get(self.tx_type)
+        if not spec_tx_class:
+            assert isinstance(extra, (bytes, bytearray))
+            return pack_varbytes(extra)
+
+        if not isinstance(extra, spec_tx_class):
+            raise ValueError('Dash tx_type does not conform with extra'
+                             ' payload class: %s, %s' % (self.tx_type, extra))
+        return pack_varbytes(extra.serialize())
+
+
+# https://github.com/dashpay/dips/blob/master/dip-0002-special-transactions.md
+@dataclass(kw_only=True, slots=True)
+class DashProRegTx:
+    '''Class representing DIP3 ProRegTx'''
+    version: int
+    type: int
+    mode: int
+    collateralOutpoint: 'TxOutPoint'
+    ipAddress: bytes
+    port: int
+    KeyIdOwner: bytes
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    operatorReward: int
+    scriptPayout: bytes
+    inputsHash: bytes
+    platformNodeID: bytes
+    platformP2PPort: int
+    platformHTTPPort: int
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.ipAddress) == 16
+                and len(self.KeyIdOwner) == 20
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        res = (
+            pack_le_uint16(self.version) +                  # version
+            pack_le_uint16(self.type) +                     # type
+            pack_le_uint16(self.mode) +                     # mode
+            self.collateralOutpoint.serialize() +           # collateralOutpoint
+            self.ipAddress +                                # ipAddress
+            pack_be_uint16(self.port) +                     # port
+            self.KeyIdOwner +                               # KeyIdOwner
+            self.PubKeyOperator +                           # PubKeyOperator
+            self.KeyIdVoting +                              # KeyIdVoting
+            pack_le_uint16(self.operatorReward) +           # operatorReward
+            pack_varbytes(self.scriptPayout) +              # scriptPayout
+            self.inputsHash)                                # inputsHash
+        if self.version >= 2:
+            res += (self.platformNodeID +                   # platformNodeID
+                    pack_le_uint16(self.platformP2PPort) +  # platformP2PPort
+                    pack_le_uint16(self.platformHTTPPort))  # platformHTTPPort
+        res += pack_varbytes(self.payloadSig)               # payloadSig
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()               # version
+        ntype = deser._read_le_uint16()                 # type
+        mode = deser._read_le_uint16()                  # mode
+        collateralOutpoint = deser._read_outpoint()     # collateralOutpoint
+        ipAddress = deser._read_nbytes(16)              # ipAddress
+        port = deser._read_be_uint16()                  # port
+        KeyIdOwner = deser._read_nbytes(20)             # KeyIdOwner
+        PubKeyOperator = deser._read_nbytes(48)         # PubKeyOperator
+        KeyIdVoting = deser._read_nbytes(20)            # KeyIdVoting
+        operatorReward = deser._read_le_uint16()        # operatorReward
+        scriptPayout = deser._read_varbytes()           # scriptPayout
+        inputsHash = deser._read_nbytes(32)             # inputsHash
+        platformNodeID = b''
+        platformP2PPort = 0
+        platformHTTPPort = 0
+        if version >= 2 and ntype == 1:
+            platformNodeID = deser._read_nbytes(20)     # platformNodeID
+            platformP2PPort = deser._read_le_uint16()   # platformP2PPort
+            platformHTTPPort = deser._read_le_uint16()  # platformHTTPPort
+        payloadSig = deser._read_varbytes()             # payloadSig
+        return DashProRegTx(
+            version=version,
+            type=ntype,
+            mode=mode,
+            collateralOutpoint=collateralOutpoint,
+            ipAddress=ipAddress,
+            port=port,
+            KeyIdOwner=KeyIdOwner,
+            PubKeyOperator=PubKeyOperator,
+            KeyIdVoting=KeyIdVoting,
+            operatorReward=operatorReward,
+            scriptPayout=scriptPayout,
+            inputsHash=inputsHash,
+            platformNodeID=platformNodeID,
+            platformP2PPort=platformP2PPort,
+            platformHTTPPort=platformHTTPPort,
+            payloadSig=payloadSig,
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashProUpServTx:
+    '''Class representing DIP3 ProUpServTx'''
+    version: int
+    type: int
+    proTxHash: bytes
+    ipAddress: bytes
+    port: int
+    scriptOperatorPayout: bytes
+    inputsHash: bytes
+    platformNodeID: bytes
+    platformP2PPort: int
+    platformHTTPPort: int
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.ipAddress) == 16
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        res = pack_le_uint16(self.version)                  # version
+        if self.version >= 2:
+            res += pack_le_uint16(self.type)                # type
+        res += (
+            self.proTxHash +                                # proTxHash
+            self.ipAddress +                                # ipAddress
+            pack_be_uint16(self.port) +                     # port
+            pack_varbytes(self.scriptOperatorPayout) +      # scriptOperatorPayout
+            self.inputsHash)                                # inputsHash
+        if self.version >= 2:
+            res += (self.platformNodeID +                   # platformNodeID
+                    pack_le_uint16(self.platformP2PPort) +  # platformP2PPort
+                    pack_le_uint16(self.platformHTTPPort))  # platformHTTPPort
+        res += self.payloadSig                              # payloadSig
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()                   # version
+        ntype = 0
+        if version >= 2:
+            ntype = deser._read_le_uint16()                 # type
+        proTxHash = deser._read_nbytes(32)                  # proTxHash
+        ipAddress = deser._read_nbytes(16)                  # ipAddress
+        port = deser._read_be_uint16()                      # port
+        scriptOperatorPayout = deser._read_varbytes()       # scriptOperatorPayout
+        inputsHash = deser._read_nbytes(32)                 # inputsHash
+        platformNodeID = b''
+        platformP2PPort = 0
+        platformHTTPPort = 0
+        if version >= 2 and ntype == 1:
+            platformNodeID = deser._read_nbytes(20)         # platformNodeID
+            platformP2PPort = deser._read_le_uint16()       # platformP2PPort
+            platformHTTPPort = deser._read_le_uint16()      # platformHTTPPort
+        payloadSig = deser._read_nbytes(96)                 # payloadSig
+        return DashProUpServTx(
+            version=version,
+            type=ntype,
+            proTxHash=proTxHash,
+            ipAddress=ipAddress,
+            port=port,
+            scriptOperatorPayout=scriptOperatorPayout,
+            inputsHash=inputsHash,
+            platformNodeID=platformNodeID,
+            platformP2PPort=platformP2PPort,
+            platformHTTPPort=platformHTTPPort,
+            payloadSig=payloadSig,
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashProUpRegTx:
+    '''Class representing DIP3 ProUpRegTx'''
+    version: int
+    proTxHash: bytes
+    mode: int
+    PubKeyOperator: bytes
+    KeyIdVoting: bytes
+    scriptPayout: bytes
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.PubKeyOperator) == 48
+                and len(self.KeyIdVoting) == 20
+                and len(self.inputsHash) == 32)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.mode) +                 # mode
+            self.PubKeyOperator +                       # PubKeyOperator
+            self.KeyIdVoting +                          # KeyIdVoting
+            pack_varbytes(self.scriptPayout) +          # scriptPayout
+            self.inputsHash +                           # inputsHash
+            pack_varbytes(self.payloadSig)              # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return DashProUpRegTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            mode=deser._read_le_uint16(),
+            PubKeyOperator=deser._read_nbytes(48),
+            KeyIdVoting=deser._read_nbytes(20),
+            scriptPayout=deser._read_varbytes(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_varbytes(),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashProUpRevTx:
+    '''Class representing DIP3 ProUpRevTx'''
+    version: int
+    proTxHash: bytes
+    reason: int
+    inputsHash: bytes
+    payloadSig: bytes
+
+    def serialize(self):
+        assert (len(self.proTxHash) == 32
+                and len(self.inputsHash) == 32
+                and len(self.payloadSig) == 96)
+        return (
+            pack_le_uint16(self.version) +              # version
+            self.proTxHash +                            # proTxHash
+            pack_le_uint16(self.reason) +               # reason
+            self.inputsHash +                           # inputsHash
+            self.payloadSig                             # payloadSig
+        )
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        return DashProUpRevTx(
+            version=deser._read_le_uint16(),
+            proTxHash=deser._read_nbytes(32),
+            reason=deser._read_le_uint16(),
+            inputsHash=deser._read_nbytes(32),
+            payloadSig=deser._read_nbytes(96),
+        )
+
+
+@dataclass(kw_only=True, slots=True)
+class DashCbTx:
+    '''Class representing DIP4 coinbase special tx'''
+    version: int
+    height: int
+    merkleRootMNList: bytes
+    merkleRootQuorums: bytes
+    bestCLHeightDiff: int
+    bestCLSignature: bytes
+    assetLockedAmount: int
+
+    def serialize(self):
+        assert len(self.merkleRootMNList) == 32
+        res = (
+            pack_le_uint16(self.version) +              # version
+            pack_le_uint32(self.height) +               # height
+            self.merkleRootMNList                       # merkleRootMNList
+        )
+        if self.version > 1:
+            assert len(self.merkleRootQuorums) == 32
+            res += self.merkleRootQuorums               # merkleRootQuorums
+        if self.version > 2:
+            res += pack_varint(self.bestCLHeightDiff)
+            res += self.bestCLSignature
+            res += pack_le_int64(self.assetLockedAmount)
+        return res
+
+    @classmethod
+    def read_tx_extra(cls, deser):
+        version = deser._read_le_uint16()
+        height = deser._read_le_uint32()
+        merkleRootMNList = deser._read_nbytes(32)
+        merkleRootQuorums = b''
+        bestCLHeightDiff = 0
+        bestCLSignature = b''
+        assetLockedAmount = 0
+        if version > 1:
+            merkleRootQuorums = deser._read_nbytes(32)
+        if version > 2:
+            bestCLHeightDiff = deser._read_varint()
+            bestCLSignature = deser._read_nbytes(96)
+            assetLockedAmount = deser._read_le_uint64()
+        return DashCbTx(
+            version=version,
+            height=height,
+            merkleRootMNList=merkleRootMNList,
+            merkleRootQuorums=merkleRootQuorums,
+            bestCLHeightDiff=bestCLHeightDiff,
+            bestCLSignature=bestCLSignature,
+            assetLockedAmount=assetLockedAmount,
+        )
+
+
+# https://dash-docs.github.io/en/developer-reference#outpoint
+@dataclass(kw_only=True, slots=True)
+class TxOutPoint:
+    '''Class representing tx output outpoint'''
+    hash: bytes
+    index: int
+
+    def serialize(self):
+        assert len(self.hash) == 32
+        return (
+            self.hash +                                 # hash
+            pack_le_uint32(self.index)                  # index
+        )
+
+    @classmethod
+    def read_outpoint(cls, deser):
+        return TxOutPoint(
+            hash=deser._read_nbytes(32),
+            index=deser._read_le_uint32(),
+        )
+
+
+class DeserializerDash(Deserializer):
+    '''Deserializer for Dash DIP2 special tx types'''
+    # Supported Spec Tx types and corresponding classes mapping
+    PRO_REG_TX = 1
+    PRO_UP_SERV_TX = 2
+    PRO_UP_REG_TX = 3
+    PRO_UP_REV_TX = 4
+    CB_TX = 5
+
+    SPEC_TX_HANDLERS = {
+        PRO_REG_TX: DashProRegTx,
+        PRO_UP_SERV_TX: DashProUpServTx,
+        PRO_UP_REG_TX: DashProUpRegTx,
+        PRO_UP_REV_TX: DashProUpRevTx,
+        CB_TX: DashCbTx,
+    }
+
+    def _read_outpoint(self):
+        return TxOutPoint.read_outpoint(self)
+
+    def read_tx(self):
+        start = self.cursor
+        header = self._read_le_uint32()
+        tx_type = header >> 16  # DIP2 tx type
+        if tx_type:
+            version = header & 0x0000ffff
+        else:
+            version = header
+
+        if tx_type and version < 3:
+            version = header
+            tx_type = 0
+
+        inputs = self._read_inputs()
+        outputs = self._read_outputs()
+        locktime = self._read_le_uint32()
+        if tx_type:
+            extra_payload_size = self._read_varint()
+            end = self.cursor + extra_payload_size
+            spec_tx_class = DeserializerDash.SPEC_TX_HANDLERS.get(tx_type)
+            if spec_tx_class:
+                read_method = getattr(spec_tx_class, 'read_tx_extra', None)
+                extra_payload = read_method(self)
+                assert isinstance(extra_payload, spec_tx_class)
+            else:
+                extra_payload = self._read_nbytes(extra_payload_size)
+            assert self.cursor == end
+        else:
+            extra_payload = b''
+        txid = self.TX_HASH_FN(self.binary[start:self.cursor])
+        tx = DashTx(
+            version=version,
+            inputs=inputs,
+            outputs=outputs,
+            locktime=locktime,
+            tx_type=tx_type,
+            extra_payload=extra_payload,
+            txid_rev=txid,
+            wtxid_rev=txid,
+        )
+        return tx

src/electrumx/lib/util.py

@@ -0,0 +1,468 @@
+# Copyright (c) 2016-2017, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+'''Miscellaneous utility classes and functions.'''
+
+
+from array import array
+import asyncio
+import inspect
+from ipaddress import ip_address
+import logging
+import sys
+from collections.abc import Container, Mapping
+from struct import Struct
+import time
+from typing import Any, Optional
+
+import aiorpcx
+
+
+# Use orjson (faster 3rd party library) if available, fallback to stdlib
+try:
+    import orjson
+    json_deserialize = orjson.loads
+
+    def json_serialize(obj: Any) -> str:
+        return orjson.dumps(obj).decode()  # orjson.dumps() returns bytes
+except ImportError:
+    import json
+    json_deserialize = json.loads
+    json_serialize = json.dumps
+
+
+# Logging utilities
+
+
+class ConnectionLogger(logging.LoggerAdapter):
+    '''Prepends a connection identifier to a logging message.'''
+    def process(self, msg, kwargs):
+        conn_id = self.extra.get('conn_id', 'unknown')
+        return f'[{conn_id}] {msg}', kwargs
+
+
+class CompactFormatter(logging.Formatter):
+    '''Strips the module from the logger name to leave the class only.'''
+    def format(self, record):
+        record.name = record.name.rpartition('.')[-1]
+        return super().format(record)
+
+
+def make_logger(name, *, handler, level):
+    '''Return the root ElectrumX logger.'''
+    logger = logging.getLogger(name)
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    logger.propagate = False
+    return logger
+
+
+def class_logger(path, classname):
+    '''Return a hierarchical logger for a class.'''
+    return logging.getLogger(path).getChild(classname)
+
+
+def formatted_time(t, sep=' '):
+    '''Return a number of seconds as a string in days, hours, mins and
+    maybe secs.'''
+    t = int(t)
+    fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60))
+    parts = []
+    for fmt, n in fmts:
+        val = t // n
+        if parts or val:
+            parts.append(fmt.format(val))
+        t %= n
+    if len(parts) < 3:
+        parts.append(f'{t:02d}s')
+    return sep.join(parts)
+
+
+def deep_getsizeof(obj):
+    """Find the memory footprint of a Python object.
+
+    Based on code from code.tutsplus.com: http://goo.gl/fZ0DXK
+
+    This is a recursive function that drills down a Python object graph
+    like a dictionary holding nested dictionaries with lists of lists
+    and tuples and sets.
+
+    The sys.getsizeof function does a shallow size of only. It counts each
+    object inside a container as pointer only regardless of how big it
+    really is.
+    """
+
+    ids = set()
+
+    def size(o):
+        if id(o) in ids:
+            return 0
+
+        r = sys.getsizeof(o)
+        ids.add(id(o))
+
+        if isinstance(o, (str, bytes, bytearray, array)):
+            return r
+
+        if isinstance(o, Mapping):
+            return r + sum(size(k) + size(v) for k, v in o.items())
+
+        if isinstance(o, Container):
+            return r + sum(size(x) for x in o)
+
+        return r
+
+    return size(obj)
+
+
+def subclasses(base_class, strict=True):
+    '''Return a list of subclasses of base_class in its module.'''
+    def select(obj):
+        return (inspect.isclass(obj) and issubclass(obj, base_class) and
+                (not strict or obj != base_class))
+
+    pairs = inspect.getmembers(sys.modules[base_class.__module__], select)
+    return [pair[1] for pair in pairs]
+
+
+def chunks(items, size):
+    '''Break up items, an iterable, into chunks of length size.'''
+    for i in range(0, len(items), size):
+        yield items[i: i + size]
+
+
+def resolve_limit(limit):
+    if limit is None or limit < 0:
+        return -1
+    assert isinstance(limit, int)
+    return limit
+
+
+def bytes_to_int(be_bytes):
+    '''Interprets a big-endian sequence of bytes as an integer'''
+    return int.from_bytes(be_bytes, 'big')
+
+
+def int_to_bytes(value):
+    '''Converts an integer to a big-endian sequence of bytes'''
+    return value.to_bytes((value.bit_length() + 7) // 8, 'big')
+
+
+def increment_byte_string(bs):
+    '''Return the lexicographically next byte string of the same length.
+
+    Return None if there is none (when the input is all 0xff bytes).'''
+    try:
+        return (int.from_bytes(bs, 'big') + 1).to_bytes(len(bs), 'big')
+    except OverflowError:
+        return None
+
+
+class LogicalFile:
+    '''A logical binary file split across several separate files on disk.'''
+
+    def __init__(self, prefix, digits, file_size):
+        self.filename_fmt = f'{prefix}{{:0{digits:d}d}}'
+        self.file_size = file_size
+
+    def read(self, start, size=-1):
+        '''Read up to size bytes from the virtual file, starting at offset
+        start, and return them.
+
+        If size is -1 all bytes are read.'''
+        parts = []
+        while size != 0:
+            try:
+                with self.open_file(start, False) as f:
+                    part = f.read(size)
+                if not part:
+                    break
+            except FileNotFoundError:
+                break
+            parts.append(part)
+            start += len(part)
+            if size > 0:
+                size -= len(part)
+        return b''.join(parts)
+
+    def write(self, start, b):
+        '''Write the bytes-like object, b, to the underlying virtual file.'''
+        while b:
+            size = min(len(b), self.file_size - (start % self.file_size))
+            with self.open_file(start, True) as f:
+                f.write(b if size == len(b) else b[:size])
+            b = b[size:]
+            start += size
+
+    def open_file(self, start, create):
+        '''Open the virtual file and seek to start.  Return a file handle.
+        Raise FileNotFoundError if the file does not exist and create
+        is False.
+        '''
+        file_num, offset = divmod(start, self.file_size)
+        filename = self.filename_fmt.format(file_num)
+        f = open_file(filename, create)
+        f.seek(offset)
+        return f
+
+
+def open_file(filename, create=False):
+    '''Open the file name.  Return its handle.'''
+    try:
+        return open(filename, 'rb+')
+    except FileNotFoundError:
+        if create:
+            return open(filename, 'wb+')
+        raise
+
+
+def open_truncate(filename):
+    '''Open the file name.  Return its handle.'''
+    return open(filename, 'wb+')
+
+
+def address_string(address):
+    '''Return an address as a correctly formatted string.'''
+    host, port = address
+    try:
+        host = ip_address(host)
+    except ValueError:
+        pass
+    else:
+        if host.version == 6:
+            return f'[{host}]:{port:d}'
+    return f'{host}:{port:d}'
+
+
+def protocol_tuple(s):
+    '''Converts a protocol version number, such as "1.0" to a tuple (1, 0).
+
+    If the version number is bad, (0, ) indicating version 0 is returned.'''
+    try:
+        return tuple(int(part) for part in s.split('.'))
+    except (TypeError, ValueError, AttributeError):
+        return (0, )
+
+
+def version_string(ptuple):
+    '''Convert a version tuple such as (1, 2) to "1.2".
+    There is always at least one dot, so (1, ) becomes "1.0".'''
+    while len(ptuple) < 2:
+        ptuple += (0, )
+    return '.'.join(str(p) for p in ptuple)
+
+
+def protocol_version(client_req, min_tuple, max_tuple):
+    '''Given a client's protocol version string, return a pair of
+    protocol tuples:
+
+           (negotiated version, client min request)
+
+    If the request is unsupported, the negotiated protocol tuple is
+    None.
+    '''
+    if client_req is None:
+        client_min = client_max = min_tuple
+    else:
+        if isinstance(client_req, list) and len(client_req) == 2:
+            client_min, client_max = client_req
+        else:
+            client_min = client_max = client_req
+        client_min = protocol_tuple(client_min)
+        client_max = protocol_tuple(client_max)
+
+    result = min(client_max, max_tuple)
+    if result < max(client_min, min_tuple) or result == (0, ):
+        result = None
+
+    return result, client_min
+
+
+def is_hex_str(text: Any, *, allow_odd_len: bool = False) -> bool:
+    if not isinstance(text, str):
+        return False
+    if allow_odd_len and len(text) % 2 == 1:
+        return is_hex_str("0" + text, allow_odd_len=False)
+    try:
+        b = bytes.fromhex(text)
+    except Exception:
+        return False
+    # forbid whitespaces in text:
+    if len(text) != 2 * len(b):
+        return False
+    return True
+
+
+struct_le_i = Struct('<i')
+struct_le_q = Struct('<q')
+struct_le_H = Struct('<H')
+struct_le_I = Struct('<I')
+struct_le_Q = Struct('<Q')
+struct_be_H = Struct('>H')
+struct_be_I = Struct('>I')
+struct_be_Q = Struct('>Q')
+structB = Struct('B')
+
+unpack_le_int32_from = struct_le_i.unpack_from
+unpack_le_int64_from = struct_le_q.unpack_from
+unpack_le_uint16_from = struct_le_H.unpack_from
+unpack_le_uint32_from = struct_le_I.unpack_from
+unpack_le_uint64_from = struct_le_Q.unpack_from
+unpack_be_uint16_from = struct_be_H.unpack_from
+unpack_be_uint32_from = struct_be_I.unpack_from
+
+unpack_le_uint32 = struct_le_I.unpack
+unpack_le_uint64 = struct_le_Q.unpack
+unpack_be_uint32 = struct_be_I.unpack
+unpack_be_uint64 = struct_be_Q.unpack
+
+pack_le_int32 = struct_le_i.pack
+pack_le_int64 = struct_le_q.pack
+pack_le_uint16 = struct_le_H.pack
+pack_le_uint32 = struct_le_I.pack
+pack_le_uint64 = struct_le_Q.pack
+pack_be_uint16 = struct_be_H.pack
+pack_be_uint32 = struct_be_I.pack
+pack_be_uint64 = struct_be_Q.pack
+pack_byte = structB.pack
+
+hex_to_bytes = bytes.fromhex
+
+
+def pack_varint(n):
+    if n < 253:
+        return pack_byte(n)
+    if n < 65536:
+        return pack_byte(253) + pack_le_uint16(n)
+    if n < 4294967296:
+        return pack_byte(254) + pack_le_uint32(n)
+    return pack_byte(255) + pack_le_uint64(n)
+
+
+def pack_varbytes(data):
+    return pack_varint(len(data)) + data
+
+
+class OldTaskGroup(aiorpcx.TaskGroup):
+    """Automatically raises exceptions on join; as in aiorpcx prior to version 0.20"""
+    async def join(self):
+        if self._wait is all:
+            exc = False
+            try:
+                async for task in self:
+                    if not task.cancelled():
+                        task.result()
+            except BaseException:  # including asyncio.CancelledError
+                exc = True
+                raise
+            finally:
+                if exc:
+                    await self.cancel_remaining()
+                await super().join()
+        else:
+            await super().join()
+            if self.completed:
+                self.completed.result()
+
+
+# We monkey-patch aiorpcx TimeoutAfter (used by timeout_after and ignore_after API),
+# to fix a timing issue present in asyncio as a whole re timing out tasks.
+# To see the issue we are trying to fix, consider example:
+#     async def outer_task():
+#         async with timeout_after(0.1):
+#             await inner_task()
+# When the 0.1 sec timeout expires, inner_task will get cancelled by timeout_after
+# (=internal cancellation).
+# If around the same time (in terms of event loop iterations) another coroutine
+# cancels outer_task (=external cancellation), there will be a race.
+# Both cancellations work by propagating a CancelledError out to timeout_after, which then
+# needs to decide (in TimeoutAfter.__aexit__) whether it's due to an internal or external cancel.
+# AFAICT asyncio provides no reliable way of distinguishing between the two.
+# This patch tries to always give priority to external cancellations.
+# see https://github.com/kyuupichan/aiorpcX/issues/44
+# see https://github.com/aio-libs/async-timeout/issues/229
+# see https://bugs.python.org/issue42130 and https://bugs.python.org/issue45098
+def _aiorpcx_monkeypatched_set_new_deadline(task, deadline):
+    def timeout_task():
+        task._orig_cancel()
+        task._timed_out = None if getattr(task, "_externally_cancelled", False) else deadline
+
+    def mycancel(*args, **kwargs):
+        task._orig_cancel(*args, **kwargs)
+        task._externally_cancelled = True
+        task._timed_out = None
+    if not hasattr(task, "_orig_cancel"):
+        task._orig_cancel = task.cancel
+        task.cancel = mycancel
+    task._deadline_handle = task._loop.call_at(deadline, timeout_task)
+
+
+def _aiorpcx_monkeypatched_set_task_deadline(task, deadline):
+    ret = _aiorpcx_orig_set_task_deadline(task, deadline)
+    task._externally_cancelled = None
+    return ret
+
+
+def _aiorpcx_monkeypatched_unset_task_deadline(task):
+    if hasattr(task, "_orig_cancel"):
+        task.cancel = task._orig_cancel
+        del task._orig_cancel
+    return _aiorpcx_orig_unset_task_deadline(task)
+
+
+_aiorpcx_orig_set_task_deadline = aiorpcx.curio._set_task_deadline
+_aiorpcx_orig_unset_task_deadline = aiorpcx.curio._unset_task_deadline
+
+aiorpcx.curio._set_new_deadline = _aiorpcx_monkeypatched_set_new_deadline
+aiorpcx.curio._set_task_deadline = _aiorpcx_monkeypatched_set_task_deadline
+aiorpcx.curio._unset_task_deadline = _aiorpcx_monkeypatched_unset_task_deadline
+
+
+def get_running_loop() -> Optional[asyncio.AbstractEventLoop]:
+    """Returns the asyncio event loop that is *running in this thread*, if any."""
+    try:
+        return asyncio.get_running_loop()
+    except RuntimeError:
+        return None
+
+
+class LogTimeTaken:
+    """A simple context manager that logs how much time passed between enter and exit."""
+
+    def __init__(self, logger: logging.Logger, base_str: str, *, enabled: bool = True):
+        self.logger = logger
+        self.base_str = base_str
+        self.enabled = enabled
+        self.start_time = 0
+
+    def __enter__(self):
+        self.start_time = time.monotonic()
+        return None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        time_elapsed = time.monotonic() - self.start_time
+        if self.enabled:
+            self.logger.debug(f'{self.base_str} took {time_elapsed:.3f}s.')