davide/plm-electrumx
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: snapshot of spesmilo/electrumx 2.0.0

Browse Source 
Unmodified source tree of upstream commit 280cb339aa158e099eb936033eecd15719770651
("Merge pull request #389 from SomberNight/202607_release_2_0",
https://github.com/spesmilo/electrumx, release 2.0.0, 3 July 2026).

All Palladium-specific changes are in the commits that follow.
This commit is contained in:
davide committed 2026-10-22 15:03:56 +02:00
commit 384d907c65
271 files changed
+27905
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/electrumx/__init__.py
+11
View File
@@ -0,0 +1,11 @@
PYTHON_MIN_VERSION = (3, 10) # noqa: E402
from electrumx.server.controller import Controller
from electrumx.server.env import Env
BRANDING = "spesmilo"
__version__ = "2.0.0"
version = f'ElectrumX {__version__}'
version_short = __version__
src/electrumx/cli/__init__.py
View File
Whitespace-only changes.
src/electrumx/cli/electrumx_rpc.py
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env python3
#
# Copyright (c) 2016-2018, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Script to send RPC commands to a running ElectrumX server.'''
import argparse
import asyncio
import json
import sys
from os import environ
from aiorpcx import timeout_after, connect_rs
import electrumx.lib.text as text
simple_commands = {
'getinfo': 'Print a summary of server state',
'groups': 'Print current session groups',
'peers': 'Print information about peer servers for the same coin',
'sessions': 'Print information about client sessions',
'stop': 'Shut down the server cleanly',
}
session_commands = {
'disconnect': 'Disconnect sessions',
'log': 'Control logging of sessions',
}
other_commands = {
'add_peer': (
'add a peer to the peers list',
[], {
'type': str,
'dest': 'real_name',
'help': 'e.g. "a.domain.name s995 t"',
},
),
'daemon_url': (
"replace the daemon's URL at run-time, and forecefully rotate "
" to the first URL in the list",
[], {
'type': str,
'nargs': '?',
'default': '',
'dest': 'daemon_url',
'help': 'see documentation of DAEMON_URL envvar',
},
),
'inspect_session': (
'inspect a given currently connected client session',
[], {
'type': int,
'dest': 'session_id',
'help': 'e.g. 1234 (see output of "sessions" command)',
},
),
'query': (
'query the UTXO and history databases',
['-l', '--limit'], {
'type': int,
'default': 1000,
'help': 'UTXO and history output limit',
}, ['items'], {
'nargs': '+',
'type': str,
'help': 'hex scripts, or addresses, to query',
},
),
'reorg': (
'simulate a chain reorganization',
[], {
'type': int,
'dest': 'count',
'default': 3,
'help': 'number of blocks to back up'
},
),
'debug_memusage_list_all_objects': (
'Print a table of types of most common types in memory',
['--limit'], {
'type': int,
'default': 50,
'help': 'max number of types to return',
},
),
'debug_memusage_get_random_backref_chain': (
"Return a dotfile as text containing the backref chain "
"for a randomly selected object of type objtype",
[], {
'type': str,
'dest': 'objtype',
'help': 'e.g. "_asyncio.Task"',
},
),
}
def main():
'''Send the RPC command to the server and print the result.'''
main_parser = argparse.ArgumentParser(
'electrumx_rpc',
description='Send electrumx an RPC command'
)
main_parser.add_argument('-p', '--port', metavar='port_num', type=int,
help='RPC port number')
main_parser.add_argument('--timeout', type=int, default=30,
help='timeout for command in seconds')
subparsers = main_parser.add_subparsers(help='sub-command help',
dest='command')
for command, help in simple_commands.items():
parser = subparsers.add_parser(command, help=help)
for command, help in session_commands.items():
parser = subparsers.add_parser(command, help=help)
parser.add_argument('session_ids', nargs='+', type=str,
help='list of session ids')
for command, data in other_commands.items():
parser_help, *arguments = data
parser = subparsers.add_parser(command, help=parser_help)
for n in range(0, len(arguments), 2):
args, kwargs = arguments[n: n+2]
parser.add_argument(*args, **kwargs)
args = main_parser.parse_args()
args = vars(args)
port = args.pop('port')
if port is None:
port = int(environ.get('RPC_PORT', 8000))
method = args.pop('command')
timeout = args.pop('timeout')
# aiorpcX makes this so easy...
async def send_request():
try:
async with timeout_after(timeout):
async with connect_rs('localhost', port) as session:
session.transport._framer.max_size = 0
session.sent_request_timeout = timeout
result = await session.send_request(method, args)
if method in ('query', ):
for line in result:
print(line)
elif method in (
'debug_memusage_list_all_objects',
'debug_memusage_get_random_backref_chain',
):
for line in result.split('\n'):
print(line)
elif method in ('groups', 'peers', 'sessions'):
lines_func = getattr(text, f'{method}_lines')
for line in lines_func(result):
print(line)
else:
print(json.dumps(result, indent=4, sort_keys=True))
return 0
except OSError:
print('cannot connect - is ElectrumX catching up, not running, or '
f'is {port} the wrong RPC port?')
return 1
except Exception as e:
print(f'error making request: {e}')
return 1
code = asyncio.run(send_request())
sys.exit(code)
if __name__ == '__main__':
main()
src/electrumx/cli/electrumx_server.py
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
#
# Copyright (c) 2016-2018, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Script to kick off the server.'''
import asyncio
import logging
import sys
import electrumx
from electrumx import Controller, Env, PYTHON_MIN_VERSION
from electrumx.lib.util import CompactFormatter, make_logger
def main():
'''Set up logging and run the server.'''
log_fmt = Env.default('LOG_FORMAT', '%(levelname)s:%(name)s:%(message)s')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(CompactFormatter(log_fmt))
logger = make_logger('electrumx', handler=handler, level='INFO')
logger.info(
f'ElectrumX server starting. '
f'({electrumx.BRANDING}. version={electrumx.__version__})')
try:
if sys.version_info < PYTHON_MIN_VERSION:
mvs = '.'.join(str(part) for part in PYTHON_MIN_VERSION)
raise RuntimeError(f'ElectrumX requires Python {mvs} or greater')
env = Env()
logger.info(f'logging level: {env.log_level}')
logger.setLevel(env.log_level)
controller = Controller(env)
asyncio.run(controller.run())
except Exception:
logger.exception('ElectrumX server terminated abnormally')
else:
logger.info('ElectrumX server terminated normally')
if __name__ == '__main__':
main()
src/electrumx/lib/__init__.py
View File
Whitespace-only changes.
src/electrumx/lib/coins.py
+3812
View File
File diff suppressed because it is too large. Load diff
src/electrumx/lib/enum.py
+54
View File
@@ -0,0 +1,54 @@
# Copyright (c) 2016, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''An enum-like type with reverse lookup.
Source: Python Cookbook, http://code.activestate.com/recipes/67107/
'''
class EnumError(Exception):
pass
class Enumeration:
def __init__(self, name, enumList):
self.__doc__ = name
lookup = {}
reverseLookup = {}
i = 0
uniqueNames = set()
uniqueValues = set()
for x in enumList:
if isinstance(x, tuple):
x, i = x
if not isinstance(x, str):
raise EnumError("enum name {} not a string".format(x))
if not isinstance(i, int):
raise EnumError("enum value {} not an integer".format(i))
if x in uniqueNames:
raise EnumError("enum name {} not unique".format(x))
if i in uniqueValues:
raise EnumError("enum value {} not unique".format(x))
uniqueNames.add(x)
uniqueValues.add(i)
lookup[x] = i
reverseLookup[i] = x
i = i + 1
self.lookup = lookup
self.reverseLookup = reverseLookup
def __getattr__(self, attr):
result = self.lookup.get(attr)
if result is None:
raise AttributeError('enumeration has no member {}'.format(attr))
return result
def whatis(self, value):
return self.reverseLookup[value]
src/electrumx/lib/env_base.py
+79
View File
@@ -0,0 +1,79 @@
# Copyright (c) 2017, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Class for server environment configuration and defaults.'''
from os import environ
from electrumx.lib.util import class_logger
class EnvBase:
'''Wraps environment configuration.'''
class Error(Exception):
pass
def __init__(self):
self.logger = class_logger(__name__, self.__class__.__name__)
self.allow_root = self.boolean('ALLOW_ROOT', False)
self.loop_policy = self.event_loop_policy()
@classmethod
def default(cls, envvar, default):
return environ.get(envvar, default)
@classmethod
def boolean(cls, envvar, default):
default = 'Yes' if default else ''
return bool(cls.default(envvar, default).strip())
@classmethod
def required(cls, envvar):
value = environ.get(envvar)
if value is None:
raise cls.Error(f'required envvar {envvar} not set')
return value
@classmethod
def integer(cls, envvar, default):
value = environ.get(envvar)
if value is None:
return default
try:
return int(value)
except Exception:
raise cls.Error(f'cannot convert envvar {envvar} value {value} to '
f'an integer')
@classmethod
def custom(cls, envvar, default, parse):
value = environ.get(envvar)
if value is None:
return default
try:
return parse(value)
except Exception as e:
raise cls.Error(
f'cannot parse envvar {envvar} value {value}'
) from e
@classmethod
def obsolete(cls, envvars):
bad = [envvar for envvar in envvars if environ.get(envvar)]
if bad:
raise cls.Error(f'remove obsolete environment variables {bad}')
def event_loop_policy(self):
policy = self.default('EVENT_LOOP_POLICY', None)
if policy is None:
return None
if policy == 'uvloop':
import uvloop
return uvloop.EventLoopPolicy()
raise self.Error(f'unknown event loop policy "{policy}"')
src/electrumx/lib/hash.py
+139
View File
@@ -0,0 +1,139 @@
# Copyright (c) 2016-2017, Neil Booth
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''Cryptograph hash functions and related classes.'''
import hashlib
import hmac
from electrumx.lib.util import bytes_to_int, int_to_bytes, hex_to_bytes
_sha256 = hashlib.sha256
_new_hash = hashlib.new
_hmac_digest = hmac.digest
HASHX_LEN = 11
def sha256(x: bytes) -> bytes:
'''Simple wrapper of hashlib sha256.'''
return _sha256(x).digest()
def double_sha256(x: bytes) -> bytes:
'''SHA-256 of SHA-256, as used extensively in bitcoin.'''
return sha256(sha256(x))
def hash_to_hex_str(x: bytes) -> str:
'''Convert a big-endian binary hash to displayed hex string. (rev-to-hum)
Display form of a binary hash is reversed and converted to hex.
'''
return bytes(reversed(x)).hex()
def hex_str_to_hash(x: str) -> bytes:
'''Convert a displayed hex string to a binary hash. (hum-to-rev)'''
return bytes(reversed(hex_to_bytes(x)))
class Base58Error(Exception):
'''Exception used for Base58 errors.'''
class Base58:
'''Class providing base 58 functionality.'''
chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
assert len(chars) == 58
cmap = {c: n for n, c in enumerate(chars)}
@staticmethod
def char_value(c):
val = Base58.cmap.get(c)
if val is None:
raise Base58Error(f'invalid base 58 character "{c}"')
return val
@staticmethod
def decode(txt):
"""Decodes txt into a big-endian bytearray."""
if not isinstance(txt, str):
raise TypeError('a string is required')
if not txt:
raise Base58Error('string cannot be empty')
value = 0
for c in txt:
value = value * 58 + Base58.char_value(c)
result = int_to_bytes(value)
# Prepend leading zero bytes if necessary
count = 0
for c in txt:
if c != '1':
break
count += 1
if count:
result = bytes(count) + result
return result
@staticmethod
def encode(be_bytes):
"""Converts a big-endian bytearray into a base58 string."""
value = bytes_to_int(be_bytes)
txt = ''
while value:
value, mod = divmod(value, 58)
txt += Base58.chars[mod]
for byte in be_bytes:
if byte != 0:
break
txt += '1'
return txt[::-1]
@staticmethod
def decode_check(txt, *, hash_fn=double_sha256):
'''Decodes a Base58Check-encoded string to a payload. The version
prefixes it.'''
be_bytes = Base58.decode(txt)
result, check = be_bytes[:-4], be_bytes[-4:]
if check != hash_fn(result)[:4]:
raise Base58Error(f'invalid base 58 checksum for {txt}')
return result
@staticmethod
def encode_check(payload, *, hash_fn=double_sha256):
"""Encodes a payload bytearray (which includes the version byte(s))
into a Base58Check string."""
be_bytes = payload + hash_fn(payload)[:4]
return Base58.encode(be_bytes)
src/electrumx/lib/lrucache.py
+190
View File
@@ -0,0 +1,190 @@
# The MIT License (MIT)
#
# Copyright (c) 2014-2022 Thomas Kemmer
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# -----
#
# This is a stripped down LRU-cache from the "cachetools" library.
# https://github.com/tkem/cachetools/blob/d991ac71b4eb6394be5ec572b835434081393215/src/cachetools/__init__.py
import collections
import collections.abc
from typing import TypeVar, Dict
class _DefaultSize:
__slots__ = ()
def __getitem__(self, _):
return 1
def __setitem__(self, _, value):
assert value == 1
def pop(self, _):
return 1
_KT = TypeVar("_KT")
_VT = TypeVar("_VT")
class Cache(collections.abc.MutableMapping[_KT, _VT]):
"""Mutable mapping to serve as a simple cache or cache base class."""
__marker = object()
__size = _DefaultSize()
def __init__(self, maxsize: int, getsizeof=None):
if getsizeof:
self.getsizeof = getsizeof
if self.getsizeof is not Cache.getsizeof:
self.__size = dict()
self.__data = dict() # type: Dict[_KT, _VT]
self.__currsize = 0
self.__maxsize = maxsize
# these can be used externally to keep track of cache hit statistics:
# (proper internal bookkeeping w/o double-counting seems either expensive or complicated)
self.num_lookups = 0
self.num_hits = 0
def __repr__(self):
return "%s(%s, maxsize=%r, currsize=%r)" % (
self.__class__.__name__,
repr(self.__data),
self.__maxsize,
self.__currsize,
)
def __getitem__(self, key: _KT) -> _VT:
try:
return self.__data[key]
except KeyError:
return self.__missing__(key)
def __setitem__(self, key: _KT, value: _VT) -> None:
maxsize = self.__maxsize
size = self.getsizeof(value)
if size > maxsize:
raise ValueError("value too large")
if key not in self.__data or self.__size[key] < size:
while self.__currsize + size > maxsize:
self.popitem()
if key in self.__data:
diffsize = size - self.__size[key]
else:
diffsize = size
self.__data[key] = value
self.__size[key] = size
self.__currsize += diffsize
def __delitem__(self, key: _KT) -> None:
size = self.__size.pop(key)
del self.__data[key]
self.__currsize -= size
def __contains__(self, key: _KT) -> bool:
return key in self.__data
def __missing__(self, key: _KT):
raise KeyError(key)
def __iter__(self):
return iter(self.__data)
def __len__(self):
return len(self.__data)
def get(self, key: _KT, default: _VT = None) -> _VT | None:
if key in self:
return self[key]
else:
return default
def pop(self, key: _KT, default=__marker) -> _VT:
if key in self:
value = self[key]
del self[key]
elif default is self.__marker:
raise KeyError(key)
else:
value = default
return value
def setdefault(self, key: _KT, default: _VT = None) -> _VT | None:
if key in self:
value = self[key]
else:
self[key] = value = default
return value
@property
def maxsize(self) -> int:
"""The maximum size of the cache."""
return self.__maxsize
@property
def currsize(self) -> int:
"""The current size of the cache."""
return self.__currsize
@staticmethod
def getsizeof(value) -> int:
"""Return the size of a cache element's value."""
return 1
class LRUCache(Cache[_KT, _VT]):
"""Least Recently Used (LRU) cache implementation."""
def __init__(self, maxsize: int, getsizeof=None):
Cache.__init__(self, maxsize, getsizeof)
self.__order = collections.OrderedDict()
def __getitem__(self, key: _KT, cache_getitem=Cache.__getitem__) -> _VT | None:
value = cache_getitem(self, key)
if key in self: # __missing__ may not store item
self.__update(key)
return value
def __setitem__(self, key: _KT, value, cache_setitem=Cache.__setitem__) -> None:
cache_setitem(self, key, value)
self.__update(key)
def __delitem__(self, key: _KT, cache_delitem=Cache.__delitem__) -> None:
cache_delitem(self, key)
del self.__order[key]
def popitem(self) -> tuple[_KT, _VT]:
"""Remove and return the `(key, value)` pair least recently used."""
try:
key = next(iter(self.__order))
except StopIteration:
raise KeyError("%s is empty" % type(self).__name__) from None
else:
return (key, self.pop(key))
def __update(self, key: _KT) -> None:
try:
self.__order.move_to_end(key)
except KeyError:
self.__order[key] = None
src/electrumx/lib/merkle.py
+266
View File
@@ -0,0 +1,266 @@
# Copyright (c) 2018, Neil Booth
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# and warranty status of this software.
'''Merkle trees, branches, proofs and roots.'''
from math import ceil, log
from typing import Sequence, Optional, Callable, Tuple, Iterable, List, Awaitable
from aiorpcx import Event, run_in_thread
from electrumx.lib.hash import double_sha256
from electrumx.lib.util import class_logger
class Merkle:
'''Perform merkle tree calculations on binary hashes using a given hash
function.
If the hash count is not even, the final hash is repeated when
calculating the next merkle layer up the tree.
'''
def __init__(self, hash_func: Callable[[bytes], bytes] = double_sha256):
self.hash_func = hash_func
def tree_depth(self, hash_count: int) -> int:
return self.branch_length(hash_count) + 1
def branch_length(self, hash_count: int) -> int:
'''Return the length of a merkle branch given the number of hashes.'''
if not isinstance(hash_count, int):
raise TypeError('hash_count must be an integer')
if hash_count < 1:
raise ValueError('hash_count must be at least 1')
return ceil(log(hash_count, 2))
def branch_and_root(
self, hashes: Iterable[bytes], index: int, length: Optional[int] = None,
) -> Tuple[List[bytes], bytes]:
'''Return a (merkle branch, merkle_root) pair given hashes, and the
index of one of those hashes.
'''
hashes = list(hashes)
if not isinstance(index, int):
raise TypeError('index must be an integer')
# This also asserts hashes is not empty
if not 0 <= index < len(hashes):
raise ValueError('index out of range')
natural_length = self.branch_length(len(hashes))
if length is None:
length = natural_length
else:
if not isinstance(length, int):
raise TypeError('length must be an integer')
if length < natural_length:
raise ValueError('length out of range')
hash_func = self.hash_func
branch = []
for _ in range(length):
if len(hashes) & 1:
hashes.append(hashes[-1])
branch.append(hashes[index ^ 1])
index >>= 1
hashes = [hash_func(hashes[n] + hashes[n + 1])
for n in range(0, len(hashes), 2)]
return branch, hashes[0]
def root(self, hashes: Sequence[bytes], length: Optional[int] = None) -> bytes:
'''Return the merkle root of a non-empty iterable of binary hashes.'''
_branch, root = self.branch_and_root(hashes, 0, length)
return root
def root_from_proof(self, hash: bytes, branch: Iterable[bytes], index: int) -> bytes:
'''Return the merkle root given a hash, a merkle branch to it, and
its index in the hashes array.
branch is an iterable sorted deepest to shallowest. If the
returned root is the expected value then the merkle proof is
verified.
The caller should have confirmed the length of the branch with
branch_length(). Unfortunately this is not easily done for
bitcoin transactions as the number of transactions in a block
is unknown to an SPV client.
'''
hash_func = self.hash_func
for elt in branch:
if index & 1:
hash = hash_func(elt + hash)
else:
hash = hash_func(hash + elt)
index >>= 1
if index:
raise ValueError('index out of range for branch')
return hash
def level(self, hashes: Sequence[bytes], depth_higher: int) -> List[bytes]:
'''Return a level of the merkle tree of hashes the given depth
higher than the bottom row of the original tree.'''
size = 1 << depth_higher
root = self.root
return [root(hashes[n: n + size], depth_higher)
for n in range(0, len(hashes), size)]
def branch_and_root_from_level(
self, level: List[bytes], leaf_hashes: Sequence[bytes], index: int, depth_higher: int,
) -> Tuple[List[bytes], bytes]:
'''Return a (merkle branch, merkle_root) pair when a merkle-tree has a
level cached.
To maximally reduce the amount of data hashed in computing a
merkle branch, cache a tree of depth N at level N // 2.
level is a list of hashes in the middle of the tree (returned
by level())
leaf_hashes are the leaves needed to calculate a partial branch
up to level.
depth_higher is how much higher level is than the leaves of the tree
index is the index in the full list of hashes of the hash whose
merkle branch we want.
'''
if not isinstance(level, list):
raise TypeError("level must be a list")
if not isinstance(leaf_hashes, list):
raise TypeError("leaf_hashes must be a list")
leaf_index = (index >> depth_higher) << depth_higher
leaf_branch, leaf_root = self.branch_and_root(
leaf_hashes, index - leaf_index, depth_higher)
index >>= depth_higher
level_branch, root = self.branch_and_root(level, index)
# Check last so that we know index is in-range
if leaf_root != level[index]:
raise ValueError('leaf hashes inconsistent with level')
return leaf_branch + level_branch, root
class MerkleCache:
'''A cache to calculate merkle branches efficiently.'''
def __init__(
self,
merkle: 'Merkle',
source_func: Callable[[int, int], Awaitable[Sequence[bytes]]],
):
'''Initialise a cache hashes taken from source_func:
async def source_func(index, count):
...
'''
self.logger = class_logger(__name__, self.__class__.__name__)
self.merkle = merkle
self.source_func = source_func
self.length = 0 # type: int
self.level = [] # type: List[bytes]
self.depth_higher = 0 # type: int
self.initialized = Event()
def _segment_length(self) -> int:
return 1 << self.depth_higher
def _leaf_start(self, index: int) -> int:
'''Given a level's depth higher and a hash index, return the leaf
index and leaf hash count needed to calculate a merkle branch.
'''
depth_higher = self.depth_higher
return (index >> depth_higher) << depth_higher
def _level(self, hashes: Sequence[bytes]) -> List[bytes]:
return self.merkle.level(hashes, self.depth_higher)
async def _extend_to(self, length: int) -> None:
'''Extend the length of the cache if necessary.'''
if length <= self.length:
return
# Start from the beginning of any final partial segment.
# Retain the value of depth_higher; in practice this is fine
start = self._leaf_start(self.length)
hashes = await self.source_func(start, length - start)
self.level[start >> self.depth_higher:] = self._level(hashes)
self.length = length
async def _level_for(self, length: int) -> List[bytes]:
'''Return a (level_length, final_hash) pair for a truncation
of the hashes to the given length.'''
if length == self.length:
return self.level
level = self.level[:length >> self.depth_higher]
leaf_start = self._leaf_start(length)
count = min(self._segment_length(), length - leaf_start)
hashes = await self.source_func(leaf_start, count)
level += self._level(hashes)
return level
async def initialize(self, length: int) -> None:
'''Call to initialize the cache to a source of given length.'''
self.length = length
self.depth_higher = self.merkle.tree_depth(length) // 2
hashes = await self.source_func(0, length)
self.level = await run_in_thread(self._level, hashes)
self.initialized.set()
def truncate(self, length: int) -> None:
'''Truncate the cache so it covers no more than length underlying
hashes.'''
if not isinstance(length, int):
raise TypeError('length must be an integer')
if length <= 0:
raise ValueError('length must be positive')
if length >= self.length:
return
length = self._leaf_start(length)
self.length = length
self.level[length >> self.depth_higher:] = []
async def branch_and_root(self, length: int, index: int) -> Tuple[List[bytes], bytes]:
'''Return a merkle branch and root. Length is the number of
hashes used to calculate the merkle root, index is the position
of the hash to calculate the branch of.
index must be less than length, which must be at least 1.'''
if not isinstance(length, int):
raise TypeError('length must be an integer')
if not isinstance(index, int):
raise TypeError('index must be an integer')
if length <= 0:
raise ValueError('length must be positive')
if index >= length:
raise ValueError('index must be less than length')
await self.initialized.wait()
await self._extend_to(length)
leaf_start = self._leaf_start(index)
count = min(self._segment_length(), length - leaf_start)
leaf_hashes = await self.source_func(leaf_start, count)
if length < self._segment_length():
return self.merkle.branch_and_root(leaf_hashes, index)
level = await self._level_for(length)
return self.merkle.branch_and_root_from_level(
level, leaf_hashes, index, self.depth_higher)
src/electrumx/lib/peer.py
+337
View File
@@ -0,0 +1,337 @@
# Copyright (c) 2017, Neil Booth
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''Representation of a peer server.'''
from functools import cached_property
from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network
from socket import AF_INET, AF_INET6
from typing import Any
from aiorpcx import is_valid_hostname
from electrumx.lib.util import protocol_tuple, version_string
class Peer:
# Protocol version
ATTRS = ('host', 'features',
# metadata
'source', 'ip_addr',
'last_good', 'last_try', 'try_count')
FEATURES = ('pruning', 'server_version', 'protocol_min', 'protocol_max',
'ssl_port', 'tcp_port')
# This should be set by the application
DEFAULT_PORTS = {}
def __init__(self, host, features, source='unknown', ip_addr=None,
last_good=0, last_try=0, try_count=0):
'''Create a peer given a host name (or IP address as a string),
a dictionary of features, and a record of the source.'''
assert isinstance(host, str)
assert isinstance(features, dict)
assert host in features.get('hosts', {})
self.host = host
self.features = features.copy()
# Canonicalize / clean-up
for feature in self.FEATURES:
self.features[feature] = getattr(self, feature)
# Metadata
self.source = source
self.ip_addr = ip_addr
# last_good represents the last connection that was
# successful *and* successfully verified, at which point
# try_count is set to 0. Failure to connect or failure to
# verify increment the try_count.
self.last_good = last_good
self.last_try = last_try
self.try_count = try_count
# Transient, non-persisted metadata
self.bad = False
self.other_port_pairs = set()
@classmethod
def peers_from_features(cls, features: dict[str, Any] | Any, source):
peers = []
if isinstance(features, dict):
hosts = features.get('hosts')
if isinstance(hosts, dict):
peers = [Peer(host, features, source=source)
for host in hosts if isinstance(host, str)]
return peers
@classmethod
def deserialize(cls, item):
'''Deserialize from a dictionary.'''
return cls(**item)
def matches(self, peers):
'''Return peers whose host matches our hostname or IP address.
Additionally include all peers whose IP address matches our
hostname if that is an IP address.
'''
candidates = (self.host.lower(), self.ip_addr)
return [peer for peer in peers
if peer.host.lower() in candidates
or peer.ip_addr == self.host]
def __str__(self):
return self.host
def update_features(self, features):
'''Update features in-place.'''
try:
tmp = Peer(self.host, features)
except AssertionError:
pass
else:
self.update_features_from_peer(tmp)
def update_features_from_peer(self, peer):
if peer != self:
self.features = peer.features
for feature in self.FEATURES:
setattr(self, feature, getattr(peer, feature))
def connection_tuples(self):
'''Return a list of (kind, port, family) tuples to try when making a
connection.
'''
# Use a list not a set - it's important to try the registered
# ports first.
pairs = [('SSL', self.ssl_port), ('TCP', self.tcp_port)]
while self.other_port_pairs:
pairs.append(self.other_port_pairs.pop())
if isinstance(self.ip_address, IPv4Address):
families = (AF_INET,)
elif isinstance(self.ip_address, IPv6Address):
families = (AF_INET6,)
else:
families = (AF_INET, AF_INET6)
return [(kind, port, family)
for kind, port in pairs if port
for family in families]
def mark_bad(self):
'''Mark as bad to avoid reconnects but also to remember for a
while.'''
self.bad = True
def check_ports(self, other):
'''Remember differing ports in case server operator changed them
or removed one.'''
if other.ssl_port != self.ssl_port:
self.other_port_pairs.add(('SSL', other.ssl_port))
if other.tcp_port != self.tcp_port:
self.other_port_pairs.add(('TCP', other.tcp_port))
return bool(self.other_port_pairs)
@cached_property
def is_tor(self):
return self.host.endswith('.onion')
@cached_property
def is_valid(self):
ip = self.ip_address
if ip:
return ((ip.is_global or ip.is_private)
and not (ip.is_multicast or ip.is_unspecified))
return is_valid_hostname(self.host)
@cached_property
def is_public(self):
ip = self.ip_address
if ip:
return self.is_valid and not ip.is_private
else:
return self.is_valid and self.host != 'localhost'
@cached_property
def ip_address(self):
'''The host as a python ip_address object, or None.'''
try:
return ip_address(self.host)
except ValueError:
return None
def bucket_for_internal_purposes(self):
'''Used for keeping the internal peer list manageable in size.
Restrictions are loose.
'''
if self.is_tor:
return 'onion'
if not self.ip_addr:
return ''
ip_addr = ip_address(self.ip_addr)
if ip_addr.version == 4:
return str(ip_addr)
elif ip_addr.version == 6:
slash64 = IPv6Network(self.ip_addr).supernet(prefixlen_diff=128-64)
return str(slash64)
return ''
def bucket_for_external_interface(self):
'''Used when responding to RPC queries to return a distributed list
of peers. Restrictions are stricter than internal bucketing.
'''
if self.is_tor:
return 'onion'
if not self.ip_addr:
return ''
ip_addr = ip_address(self.ip_addr)
if ip_addr.version == 4:
slash16 = IPv4Network(self.ip_addr).supernet(prefixlen_diff=32-16)
return str(slash16)
elif ip_addr.version == 6:
slash56 = IPv6Network(self.ip_addr).supernet(prefixlen_diff=128-56)
return str(slash56)
return ''
def serialize(self):
'''Serialize to a dictionary.'''
return {attr: getattr(self, attr) for attr in self.ATTRS}
def _port(self, key):
hosts = self.features.get('hosts')
if isinstance(hosts, dict):
host = hosts.get(self.host)
port = self._integer(key, host)
if port and 0 < port < 65536:
return port
return None
def _integer(self, key, d=None):
d = d or self.features
result = d.get(key) if isinstance(d, dict) else None
if isinstance(result, str):
try:
result = int(result)
except ValueError:
pass
return result if isinstance(result, int) else None
def _string(self, key):
result = self.features.get(key)
return result if isinstance(result, str) else None
@cached_property
def genesis_hash(self):
'''Returns the network genesis block hash as a string if known, otherwise None.'''
return self._string('genesis_hash')
@cached_property
def ssl_port(self):
'''Returns None if no SSL port, otherwise the port as an integer.'''
return self._port('ssl_port')
@cached_property
def tcp_port(self):
'''Returns None if no TCP port, otherwise the port as an integer.'''
return self._port('tcp_port')
@cached_property
def server_version(self):
'''Returns the server version as a string if known, otherwise None.'''
return self._string('server_version')
@cached_property
def pruning(self):
'''Returns the pruning level as an integer. None indicates no
pruning.'''
pruning = self._integer('pruning')
if pruning and pruning > 0:
return pruning
return None
def _protocol_version_string(self, key):
version_str = self.features.get(key)
ptuple = protocol_tuple(version_str)
return version_string(ptuple)
@cached_property
def protocol_min(self):
'''Minimum protocol version as a string, e.g., 1.0'''
return self._protocol_version_string('protocol_min')
@cached_property
def protocol_max(self):
'''Maximum protocol version as a string, e.g., 1.1'''
return self._protocol_version_string('protocol_max')
def to_tuple(self):
'''The tuple ((ip, host, details) expected in response
to a peers subscription.'''
details = self.real_name().split()[1:]
return (self.ip_addr or self.host, self.host, details)
def real_name(self):
'''Real name of this peer as used on IRC.'''
def port_text(letter, port):
if port == self.DEFAULT_PORTS.get(letter):
return letter
else:
return letter + str(port)
parts = [self.host, 'v' + self.protocol_max]
if self.pruning:
parts.append(f'p{self.pruning:d}')
for letter, port in (('s', self.ssl_port), ('t', self.tcp_port)):
if port:
parts.append(port_text(letter, port))
return ' '.join(parts)
@classmethod
def from_real_name(cls, real_name, source):
'''Real name is a real name as on IRC, such as
"erbium1.sytes.net v1.0 s t"
Returns an instance of this Peer class.
'''
host = 'nohost'
features = {}
ports = {}
for n, part in enumerate(real_name.split()):
if n == 0:
host = part
continue
if part[0] in ('s', 't'):
if len(part) == 1:
port = cls.DEFAULT_PORTS[part[0]]
else:
port = part[1:]
if part[0] == 's':
ports['ssl_port'] = port
else:
ports['tcp_port'] = port
elif part[0] == 'v':
features['protocol_max'] = features['protocol_min'] = part[1:]
elif part[0] == 'p':
features['pruning'] = part[1:]
features.update(ports)
features['hosts'] = {host: ports}
return cls(host, features, source)
src/electrumx/lib/script.py
+191
View File
@@ -0,0 +1,191 @@
# Copyright (c) 2016-2017, Neil Booth
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# and warranty status of this software.
'''Script-related classes and functions.'''
from electrumx.lib.enum import Enumeration
from electrumx.lib.util import unpack_le_uint16_from, unpack_le_uint32_from, \
pack_le_uint16, pack_le_uint32
class ScriptError(Exception):
'''Exception used for script errors.'''
OpCodes = Enumeration("Opcodes", [
("OP_0", 0), ("OP_PUSHDATA1", 76),
"OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE",
"OP_RESERVED",
"OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7", "OP_8",
"OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16",
"OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF",
"OP_ELSE", "OP_ENDIF", "OP_VERIFY", "OP_RETURN",
"OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP",
"OP_2OVER", "OP_2ROT", "OP_2SWAP", "OP_IFDUP", "OP_DEPTH", "OP_DROP",
"OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL", "OP_ROT",
"OP_SWAP", "OP_TUCK",
"OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE",
"OP_INVERT", "OP_AND", "OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY",
"OP_RESERVED1", "OP_RESERVED2",
"OP_1ADD", "OP_1SUB", "OP_2MUL", "OP_2DIV", "OP_NEGATE", "OP_ABS",
"OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL", "OP_DIV", "OP_MOD",
"OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR", "OP_NUMEQUAL",
"OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN", "OP_GREATERTHAN",
"OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX",
"OP_WITHIN",
"OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160", "OP_HASH256",
"OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG",
"OP_CHECKMULTISIGVERIFY",
"OP_NOP1",
"OP_CHECKLOCKTIMEVERIFY", "OP_CHECKSEQUENCEVERIFY"
])
# Paranoia to make it hard to create bad scripts
assert OpCodes.OP_DUP == 0x76
assert OpCodes.OP_HASH160 == 0xa9
assert OpCodes.OP_EQUAL == 0x87
assert OpCodes.OP_EQUALVERIFY == 0x88
assert OpCodes.OP_CHECKSIG == 0xac
assert OpCodes.OP_CHECKMULTISIG == 0xae
def is_unspendable_legacy(script):
# OP_FALSE OP_RETURN or OP_RETURN
return script[:2] == b'\x00\x6a' or (script and script[0] == 0x6a)
def is_unspendable_genesis(script):
# OP_FALSE OP_RETURN
return script[:2] == b'\x00\x6a'
def _match_ops(ops, pattern):
if len(ops) != len(pattern):
return False
for op, pop in zip(ops, pattern):
if pop != op:
# -1 means 'data push', whose op is an (op, data) tuple
if pop == -1 and isinstance(op, tuple):
continue
return False
return True
class ScriptPubKey:
'''A class for handling a tx output script that gives conditions
necessary for spending.
'''
TO_ADDRESS_OPS = (OpCodes.OP_DUP, OpCodes.OP_HASH160, -1,
OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG)
TO_P2SH_OPS = (OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL)
TO_PUBKEY_OPS = (-1, OpCodes.OP_CHECKSIG)
@classmethod
def P2SH_script(cls, hash160):
return (bytes((OpCodes.OP_HASH160,))
+ Script.push_data(hash160)
+ bytes((OpCodes.OP_EQUAL,)))
@classmethod
def P2PKH_script(cls, hash160):
return (bytes((OpCodes.OP_DUP, OpCodes.OP_HASH160))
+ Script.push_data(hash160)
+ bytes((OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG)))
class Script:
@classmethod
def get_ops(cls, script):
ops = []
# The unpacks or script[n] below throw on truncated scripts
try:
n = 0
while n < len(script):
op = script[n]
n += 1
if op <= OpCodes.OP_PUSHDATA4:
# Raw bytes follow
if op < OpCodes.OP_PUSHDATA1:
dlen = op
elif op == OpCodes.OP_PUSHDATA1:
dlen = script[n]
n += 1
elif op == OpCodes.OP_PUSHDATA2:
dlen, = unpack_le_uint16_from(script[n: n + 2])
n += 2
else:
dlen, = unpack_le_uint32_from(script[n: n + 4])
n += 4
if n + dlen > len(script):
raise IndexError
op = (op, script[n:n + dlen])
n += dlen
ops.append(op)
except Exception:
# Truncated script; e.g. tx_hash
# ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
raise ScriptError('truncated script')
return ops
@classmethod
def push_data(cls, data):
'''Returns the opcodes to push the data on the stack.'''
assert isinstance(data, (bytes, bytearray))
n = len(data)
if n < OpCodes.OP_PUSHDATA1:
return bytes((n,)) + data
if n < 256:
return bytes((OpCodes.OP_PUSHDATA1, n)) + data
if n < 65536:
return bytes((OpCodes.OP_PUSHDATA2,)) + pack_le_uint16(n) + data
return bytes((OpCodes.OP_PUSHDATA4,)) + pack_le_uint32(n) + data
@classmethod
def opcode_name(cls, opcode):
if OpCodes.OP_0 < opcode < OpCodes.OP_PUSHDATA1:
return f'OP_{opcode:d}'
try:
return OpCodes.whatis(opcode)
except KeyError:
return f'OP_UNKNOWN:{opcode:d}'
@classmethod
def dump(cls, script):
opcodes, datas = cls.get_ops(script)
for opcode, data in zip(opcodes, datas):
name = cls.opcode_name(opcode)
if data is None:
print(name)
else:
print(f'{name} {data.hex()} ({len(data):d} bytes)')
src/electrumx/lib/server_base.py
+131
View File
@@ -0,0 +1,131 @@
# Copyright (c) 2017, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Base class of servers'''
import asyncio
import os
import platform
import re
import signal
import sys
import time
from contextlib import suppress
from functools import partial
from typing import TYPE_CHECKING
from aiorpcx import spawn
from electrumx import PYTHON_MIN_VERSION
from electrumx.lib.util import class_logger
if TYPE_CHECKING:
from electrumx.server.env import Env
class ServerBase:
'''Base class server implementation.
Derived classes are expected to:
- set SUPPRESS_MESSAGE_REGEX as appropriate
- implement the serve() coroutine, called from the run() method.
Upon return the event loop runs until the shutdown signal is received.
'''
SUPPRESS_MESSAGE_REGEX = re.compile('SSL handshake|Fatal read error on|'
'SSL error in data received|'
'socket.send() raised exception')
SUPPRESS_TASK_REGEX = re.compile('accept_connection2')
def __init__(self, env: 'Env'):
'''Save the environment, perform basic sanity checks, and set the
event loop policy.
'''
# First asyncio operation must be to set the event loop policy
# as this replaces the event loop
asyncio.set_event_loop_policy(env.loop_policy)
self.logger = class_logger(__name__, self.__class__.__name__)
version_str = ' '.join(sys.version.splitlines())
self.logger.info(f'Python version: {version_str}')
self.env = env
self.start_time = 0
# Sanity checks
if sys.version_info < PYTHON_MIN_VERSION:
mvs = '.'.join(str(part) for part in PYTHON_MIN_VERSION)
raise RuntimeError(f'Python version >= {mvs} is required')
if platform.system() == 'Windows':
pass
elif os.geteuid() == 0 and not env.allow_root:
raise RuntimeError('RUNNING AS ROOT IS STRONGLY DISCOURAGED!\n'
'You shoud create an unprivileged user account '
'and use that.\n'
'To continue as root anyway, restart with '
'environment variable ALLOW_ROOT non-empty')
async def serve(self, shutdown_event: asyncio.Event):
'''Override to provide the main server functionality.
Run as a task that will be cancelled to request shutdown.
Setting the event also shuts down the server.
'''
def on_exception(self, loop, context):
'''Suppress spurious messages it appears we cannot control.'''
message = context.get('message')
if message and self.SUPPRESS_MESSAGE_REGEX.match(message):
return
if self.SUPPRESS_TASK_REGEX.match(repr(context.get('task'))):
return
loop.default_exception_handler(context)
async def run(self):
'''Run the server application:
- record start time
- install SIGINT and SIGTERM handlers to trigger shutdown_event
- set loop's exception handler to suppress unwanted messages
- run the event loop until serve() completes
'''
def on_signal(signame):
shutdown_event.set()
self.logger.warning(f'received {signame} signal, initiating shutdown')
async def serve():
try:
await self.serve(shutdown_event)
finally:
shutdown_event.set()
self.start_time = time.time()
loop = asyncio.get_event_loop()
shutdown_event = asyncio.Event()
if platform.system() != 'Windows':
# No signals on Windows
for signame in ('SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, signame),
partial(on_signal, signame))
loop.set_exception_handler(self.on_exception)
# Start serving and wait for shutdown, log receipt of the event
server_task = await spawn(serve, daemon=True)
try:
await shutdown_event.wait()
except KeyboardInterrupt:
self.logger.warning('received keyboard interrupt, initiating shutdown')
self.logger.info('shutting down')
server_task.cancel()
try:
with suppress(asyncio.CancelledError):
await server_task
finally:
self.logger.info('shutdown complete')
src/electrumx/lib/text.py
+86
View File
@@ -0,0 +1,86 @@
import time
import electrumx.lib.util as util
def sessions_lines(data):
'''A generator returning lines for a list of sessions.
data is the return value of rpc_sessions().'''
fmt = ('{:<6} {:<5} {:>17} {:>5} '
'{:>7} {:>7} {:>5} {:>5} {:>7} '
'{:>7} {:>7} {:>7} {:>7} {:>9} {:>21}')
yield fmt.format('ID', 'Flags', 'Client', 'Proto',
'Cost', 'XCost', 'Reqs', 'Txs', 'Subs',
'Recv', 'Recv KB', 'Sent', 'Sent KB', 'Time', 'Peer')
for (id_, flags, peer, client, proto, cost, extra_cost, reqs, txs_sent, subs,
recv_count, recv_size, send_count, send_size, conn_time) in data:
yield fmt.format(id_, flags, client, proto,
f'{int(cost):,d}',
f'{int(extra_cost):,d}',
f'{reqs:,d}',
f'{txs_sent:,d}',
f'{subs:,d}',
f'{recv_count:,d}',
f'{recv_size // 1024:,d}',
f'{send_count:,d}',
f'{send_size // 1024:,d}',
util.formatted_time(conn_time, sep=''), peer)
def groups_lines(data):
'''A generator returning lines for a list of groups.
data is the return value of rpc_groups().'''
fmt = ('{:<14} {:>9} {:>8} {:>8} {:>6} {:>6} {:>8}'
'{:>7} {:>9} {:>7} {:>9}')
yield fmt.format('Name', 'Sessions', 'Cost', 'Retained', 'Reqs', 'Txs', 'Subs',
'Recv', 'Recv KB', 'Sent', 'Sent KB')
for (name, session_count, cost, retained_cost, reqs, txs_sent, subs,
recv_count, recv_size, send_count, send_size) in data:
yield fmt.format(name,
f'{session_count:,d}',
f'{int(cost):,d}',
f'{int(retained_cost):,d}',
f'{reqs:,d}',
f'{txs_sent:,d}',
f'{subs:,d}',
f'{recv_count:,d}',
f'{recv_size // 1024:,d}',
f'{send_count:,d}',
f'{send_size // 1024:,d}')
def peers_lines(data):
'''A generator returning lines for a list of peers.
data is the return value of rpc_peers().'''
def time_fmt(t):
if not t:
return 'Never'
return util.formatted_time(now - t)
now = time.time()
fmt = ('{:<62} {:<6} {:>5} {:>5} {:<17} {:>4} '
'{:>4} {:>8} {:>11} {:>11} {:>5} {:>20} {:<15}')
yield fmt.format('Host', 'Status', 'TCP', 'SSL', 'Server', 'Min',
'Max', 'Pruning', 'Last Good', 'Last Try',
'Tries', 'Source', 'IP Address')
for item in data:
features = item['features']
hostname = item['host']
host = features['hosts'][hostname]
yield fmt.format(hostname[:62],
item['status'],
host.get('tcp_port') or '',
host.get('ssl_port') or '',
features['server_version'] or 'unknown',
features['protocol_min'],
features['protocol_max'],
features['pruning'] or '',
time_fmt(item['last_good']),
time_fmt(item['last_try']),
item['try_count'],
item['source'][:20],
item['ip_addr'] or '')
src/electrumx/lib/tx.py
+1460
View File
File diff suppressed because it is too large. Load diff
src/electrumx/lib/tx_axe.py
+489
View File
@@ -0,0 +1,489 @@
# Copyright (c) 2016-2018, Neil Booth
# Copyright (c) 2018, the ElectrumX authors
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''Deserializer for AXE DIP2 special transaction types'''
from dataclasses import dataclass
from typing import Any
from electrumx.lib.tx import Deserializer, Tx
from electrumx.lib.util import (pack_le_uint16, pack_le_int32, pack_le_uint32,
pack_le_int64, pack_varint, pack_varbytes,
pack_be_uint16)
# https://github.com/dashpay/dips/blob/master/dip-0002.md
@dataclass(kw_only=True, slots=True)
class AxeTx(Tx):
'''Class representing a Axe transaction'''
tx_type: int
extra_payload: Any
def serialize(self):
nLocktime = pack_le_uint32(self.locktime)
txins = (pack_varint(len(self.inputs)) +
b''.join(tx_in.serialize() for tx_in in self.inputs))
txouts = (pack_varint(len(self.outputs)) +
b''.join(tx_out.serialize() for tx_out in self.outputs))
if self.tx_type:
uVersion = pack_le_uint16(self.version)
uTxType = pack_le_uint16(self.tx_type)
vExtra = self._serialize_extra_payload()
return uVersion + uTxType + txins + txouts + nLocktime + vExtra
else:
nVersion = pack_le_int32(self.version)
return nVersion + txins + txouts + nLocktime
def _serialize_extra_payload(self):
extra = self.extra_payload
spec_tx_class = DeserializerAxe.SPEC_TX_HANDLERS.get(self.tx_type)
if not spec_tx_class:
assert isinstance(extra, (bytes, bytearray))
return pack_varbytes(extra)
if not isinstance(extra, spec_tx_class):
raise ValueError('Axe tx_type does not conform with extra'
' payload class: %s, %s' % (self.tx_type, extra))
return pack_varbytes(extra.serialize())
# https://github.com/dashpay/dips/blob/master/dip-0002-special-transactions.md
@dataclass(kw_only=True, slots=True)
class AxeProRegTx:
'''Class representing DIP3 ProRegTx'''
version: int
type: int
mode: int
collateralOutpoint: 'TxOutPoint'
ipAddress: bytes
port: int
KeyIdOwner: bytes
PubKeyOperator: bytes
KeyIdVoting: bytes
operatorReward: int
scriptPayout: bytes
inputsHash: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.ipAddress) == 16
and len(self.KeyIdOwner) == 20
and len(self.PubKeyOperator) == 48
and len(self.KeyIdVoting) == 20
and len(self.inputsHash) == 32)
return (
pack_le_uint16(self.version) + # version
pack_le_uint16(self.type) + # type
pack_le_uint16(self.mode) + # mode
self.collateralOutpoint.serialize() + # collateralOutpoint
self.ipAddress + # ipAddress
pack_be_uint16(self.port) + # port
self.KeyIdOwner + # KeyIdOwner
self.PubKeyOperator + # PubKeyOperator
self.KeyIdVoting + # KeyIdVoting
pack_le_uint16(self.operatorReward) + # operatorReward
pack_varbytes(self.scriptPayout) + # scriptPayout
self.inputsHash + # inputsHash
pack_varbytes(self.payloadSig) # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeProRegTx(
version=deser._read_le_uint16(),
type=deser._read_le_uint16(),
mode=deser._read_le_uint16(),
collateralOutpoint=deser._read_outpoint(),
ipAddress=deser._read_nbytes(16),
port=deser._read_be_uint16(),
KeyIdOwner=deser._read_nbytes(20),
PubKeyOperator=deser._read_nbytes(48),
KeyIdVoting=deser._read_nbytes(20),
operatorReward=deser._read_le_uint16(),
scriptPayout=deser._read_varbytes(),
inputsHash=deser._read_nbytes(32),
payloadSig=deser._read_varbytes(),
)
@dataclass(kw_only=True, slots=True)
class AxeProUpServTx:
'''Class representing DIP3 ProUpServTx'''
version: int
proTxHash: bytes
ipAddress: bytes
port: int
scriptOperatorPayout: bytes
inputsHash: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.ipAddress) == 16
and len(self.inputsHash) == 32
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
self.proTxHash + # proTxHash
self.ipAddress + # ipAddress
pack_be_uint16(self.port) + # port
pack_varbytes(self.scriptOperatorPayout) + # scriptOperatorPayout
self.inputsHash + # inputsHash
self.payloadSig # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeProUpServTx(
version=deser._read_le_uint16(),
proTxHash=deser._read_nbytes(32),
ipAddress=deser._read_nbytes(16),
port=deser._read_be_uint16(),
scriptOperatorPayout=deser._read_varbytes(),
inputsHash=deser._read_nbytes(32),
payloadSig=deser._read_nbytes(96),
)
@dataclass(kw_only=True, slots=True)
class AxeProUpRegTx:
'''Class representing DIP3 ProUpRegTx'''
version: int
proTxHash: bytes
mode: int
PubKeyOperator: bytes
KeyIdVoting: bytes
scriptPayout: bytes
inputsHash: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.PubKeyOperator) == 48
and len(self.KeyIdVoting) == 20
and len(self.inputsHash) == 32)
return (
pack_le_uint16(self.version) + # version
self.proTxHash + # proTxHash
pack_le_uint16(self.mode) + # mode
self.PubKeyOperator + # PubKeyOperator
self.KeyIdVoting + # KeyIdVoting
pack_varbytes(self.scriptPayout) + # scriptPayout
self.inputsHash + # inputsHash
pack_varbytes(self.payloadSig) # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeProUpRegTx(
version=deser._read_le_uint16(),
proTxHash=deser._read_nbytes(32),
mode=deser._read_le_uint16(),
PubKeyOperator=deser._read_nbytes(48),
KeyIdVoting=deser._read_nbytes(20),
scriptPayout=deser._read_varbytes(),
inputsHash=deser._read_nbytes(32),
payloadSig=deser._read_varbytes(),
)
@dataclass(kw_only=True, slots=True)
class AxeProUpRevTx:
'''Class representing DIP3 ProUpRevTx'''
version: int
proTxHash: bytes
reason: int
inputsHash: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.inputsHash) == 32
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
self.proTxHash + # proTxHash
pack_le_uint16(self.reason) + # reason
self.inputsHash + # inputsHash
self.payloadSig # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeProUpRevTx(
version=deser._read_le_uint16(),
proTxHash=deser._read_nbytes(32),
reason=deser._read_le_uint16(),
inputsHash=deser._read_nbytes(32),
payloadSig=deser._read_nbytes(96),
)
@dataclass(kw_only=True, slots=True)
class AxeCbTx:
'''Class representing DIP4 coinbase special tx'''
version: int
height: int
merkleRootMNList: bytes
merkleRootQuorums: bytes
def serialize(self):
assert len(self.merkleRootMNList) == 32
res = (
pack_le_uint16(self.version) + # version
pack_le_uint32(self.height) + # height
self.merkleRootMNList # merkleRootMNList
)
if self.version > 1:
assert len(self.merkleRootQuorums) == 32
res += self.merkleRootQuorums # merkleRootQuorums
return res
@classmethod
def read_tx_extra(cls, deser):
version = deser._read_le_uint16()
height = deser._read_le_uint32()
merkleRootMNList = deser._read_nbytes(32)
merkleRootQuorums = b''
if version > 1:
merkleRootQuorums = deser._read_nbytes(32)
return AxeCbTx(
version=version,
height=height,
merkleRootMNList=merkleRootMNList,
merkleRootQuorums=merkleRootQuorums,
)
@dataclass(kw_only=True, slots=True)
class AxeSubTxRegister:
'''Class representing DIP5 SubTxRegister'''
version: int
userName: bytes
pubKey: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.pubKey) == 48
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
pack_varbytes(self.userName) + # userName
self.pubKey + # pubKey
self.payloadSig # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeSubTxRegister(
version=deser._read_le_uint16(),
userName=deser._read_varbytes(),
pubKey=deser._read_nbytes(48),
payloadSig=deser._read_nbytes(96),
)
@dataclass(kw_only=True, slots=True)
class AxeSubTxTopup:
'''Class representing DIP5 SubTxTopup'''
version: int
regTxHash: bytes
def serialize(self):
assert len(self.regTxHash) == 32
return (
pack_le_uint16(self.version) + # version
self.regTxHash # regTxHash
)
@classmethod
def read_tx_extra(cls, deser):
return AxeSubTxTopup(
version=deser._read_le_uint16(),
regTxHash=deser._read_nbytes(32),
)
@dataclass(kw_only=True, slots=True)
class AxeSubTxResetKey:
'''Class representing DIP5 SubTxResetKey'''
version: int
regTxHash: bytes
hashPrevSubTx: bytes
creditFee: int
newPubKey: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.regTxHash) == 32
and len(self.hashPrevSubTx) == 32
and len(self.newPubKey) == 48
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
self.regTxHash + # regTxHash
self.hashPrevSubTx + # hashPrevSubTx
pack_le_int64(self.creditFee) + # creditFee
self.newPubKey + # newPubKey
self.payloadSig # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeSubTxResetKey(
version=deser._read_le_uint16(),
regTxHash=deser._read_nbytes(32),
hashPrevSubTx=deser._read_nbytes(32),
creditFee=deser._read_le_int64(),
newPubKey=deser._read_nbytes(48),
payloadSig=deser._read_nbytes(96),
)
@dataclass(kw_only=True, slots=True)
class AxeSubTxCloseAccount:
'''Class representing DIP5 SubTxCloseAccount'''
version: int
regTxHash: bytes
hashPrevSubTx: bytes
creditFee: int
payloadSig: bytes
def serialize(self):
assert (len(self.regTxHash) == 32
and len(self.hashPrevSubTx) == 32
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
self.regTxHash + # regTxHash
self.hashPrevSubTx + # hashPrevSubTx
pack_le_int64(self.creditFee) + # creditFee
self.payloadSig # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return AxeSubTxCloseAccount(
version=deser._read_le_uint16(),
regTxHash=deser._read_nbytes(32),
hashPrevSubTx=deser._read_nbytes(32),
creditFee=deser._read_le_int64(),
payloadSig=deser._read_nbytes(96),
)
# https://dash-docs.github.io/en/developer-reference#outpoint
@dataclass(kw_only=True, slots=True)
class TxOutPoint:
'''Class representing tx output outpoint'''
hash: bytes
index: int
def serialize(self):
assert len(self.hash) == 32
return (
self.hash + # hash
pack_le_uint32(self.index) # index
)
@classmethod
def read_outpoint(cls, deser):
return TxOutPoint(
hash=deser._read_nbytes(32),
index=deser._read_le_uint32(),
)
class DeserializerAxe(Deserializer):
'''Deserializer for AXE DIP2 special tx types'''
# Supported Spec Tx types and corresponding classes mapping
PRO_REG_TX = 1
PRO_UP_SERV_TX = 2
PRO_UP_REG_TX = 3
PRO_UP_REV_TX = 4
CB_TX = 5
SUB_TX_REGISTER = 8
SUB_TX_TOPUP = 9
SUB_TX_RESET_KEY = 10
SUB_TX_CLOSE_ACCOUNT = 11
SPEC_TX_HANDLERS = {
PRO_REG_TX: AxeProRegTx,
PRO_UP_SERV_TX: AxeProUpServTx,
PRO_UP_REG_TX: AxeProUpRegTx,
PRO_UP_REV_TX: AxeProUpRevTx,
CB_TX: AxeCbTx,
SUB_TX_REGISTER: AxeSubTxRegister,
SUB_TX_TOPUP: AxeSubTxTopup,
SUB_TX_RESET_KEY: AxeSubTxResetKey,
SUB_TX_CLOSE_ACCOUNT: AxeSubTxCloseAccount,
}
def _read_outpoint(self):
return TxOutPoint.read_outpoint(self)
def read_tx(self):
start = self.cursor
header = self._read_le_uint32()
tx_type = header >> 16 # DIP2 tx type
if tx_type:
version = header & 0x0000ffff
else:
version = header
if tx_type and version < 3:
version = header
tx_type = 0
inputs = self._read_inputs()
outputs = self._read_outputs()
locktime = self._read_le_uint32()
if tx_type:
extra_payload_size = self._read_varint()
end = self.cursor + extra_payload_size
spec_tx_class = DeserializerAxe.SPEC_TX_HANDLERS.get(tx_type)
if spec_tx_class:
read_method = getattr(spec_tx_class, 'read_tx_extra', None)
extra_payload = read_method(self)
assert isinstance(extra_payload, spec_tx_class)
else:
extra_payload = self._read_nbytes(extra_payload_size)
assert self.cursor == end
else:
extra_payload = b''
txid = self.TX_HASH_FN(self.binary[start:self.cursor])
tx = AxeTx(
version=version,
inputs=inputs,
outputs=outputs,
locktime=locktime,
tx_type=tx_type,
extra_payload=extra_payload,
txid_rev=txid,
wtxid_rev=txid,
)
return tx
src/electrumx/lib/tx_dash.py
+442
View File
@@ -0,0 +1,442 @@
# Copyright (c) 2016-2018, Neil Booth
# Copyright (c) 2018, the ElectrumX authors
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
'''Deserializer for Dash DIP2 special transaction types'''
from dataclasses import dataclass
from typing import Any
from electrumx.lib.tx import Deserializer, Tx
from electrumx.lib.util import (pack_le_uint16, pack_le_int32, pack_le_uint32,
pack_le_int64, pack_varint, pack_varbytes,
pack_be_uint16)
# https://github.com/dashpay/dips/blob/master/dip-0002.md
@dataclass(kw_only=True, slots=True)
class DashTx(Tx):
'''Class representing a Dash transaction'''
tx_type: int
extra_payload: Any
def serialize(self):
nLocktime = pack_le_uint32(self.locktime)
txins = (pack_varint(len(self.inputs)) +
b''.join(tx_in.serialize() for tx_in in self.inputs))
txouts = (pack_varint(len(self.outputs)) +
b''.join(tx_out.serialize() for tx_out in self.outputs))
if self.tx_type:
uVersion = pack_le_uint16(self.version)
uTxType = pack_le_uint16(self.tx_type)
vExtra = self._serialize_extra_payload()
return uVersion + uTxType + txins + txouts + nLocktime + vExtra
else:
nVersion = pack_le_int32(self.version)
return nVersion + txins + txouts + nLocktime
def _serialize_extra_payload(self):
extra = self.extra_payload
spec_tx_class = DeserializerDash.SPEC_TX_HANDLERS.get(self.tx_type)
if not spec_tx_class:
assert isinstance(extra, (bytes, bytearray))
return pack_varbytes(extra)
if not isinstance(extra, spec_tx_class):
raise ValueError('Dash tx_type does not conform with extra'
' payload class: %s, %s' % (self.tx_type, extra))
return pack_varbytes(extra.serialize())
# https://github.com/dashpay/dips/blob/master/dip-0002-special-transactions.md
@dataclass(kw_only=True, slots=True)
class DashProRegTx:
'''Class representing DIP3 ProRegTx'''
version: int
type: int
mode: int
collateralOutpoint: 'TxOutPoint'
ipAddress: bytes
port: int
KeyIdOwner: bytes
PubKeyOperator: bytes
KeyIdVoting: bytes
operatorReward: int
scriptPayout: bytes
inputsHash: bytes
platformNodeID: bytes
platformP2PPort: int
platformHTTPPort: int
payloadSig: bytes
def serialize(self):
assert (len(self.ipAddress) == 16
and len(self.KeyIdOwner) == 20
and len(self.PubKeyOperator) == 48
and len(self.KeyIdVoting) == 20
and len(self.inputsHash) == 32)
res = (
pack_le_uint16(self.version) + # version
pack_le_uint16(self.type) + # type
pack_le_uint16(self.mode) + # mode
self.collateralOutpoint.serialize() + # collateralOutpoint
self.ipAddress + # ipAddress
pack_be_uint16(self.port) + # port
self.KeyIdOwner + # KeyIdOwner
self.PubKeyOperator + # PubKeyOperator
self.KeyIdVoting + # KeyIdVoting
pack_le_uint16(self.operatorReward) + # operatorReward
pack_varbytes(self.scriptPayout) + # scriptPayout
self.inputsHash) # inputsHash
if self.version >= 2:
res += (self.platformNodeID + # platformNodeID
pack_le_uint16(self.platformP2PPort) + # platformP2PPort
pack_le_uint16(self.platformHTTPPort)) # platformHTTPPort
res += pack_varbytes(self.payloadSig) # payloadSig
return res
@classmethod
def read_tx_extra(cls, deser):
version = deser._read_le_uint16() # version
ntype = deser._read_le_uint16() # type
mode = deser._read_le_uint16() # mode
collateralOutpoint = deser._read_outpoint() # collateralOutpoint
ipAddress = deser._read_nbytes(16) # ipAddress
port = deser._read_be_uint16() # port
KeyIdOwner = deser._read_nbytes(20) # KeyIdOwner
PubKeyOperator = deser._read_nbytes(48) # PubKeyOperator
KeyIdVoting = deser._read_nbytes(20) # KeyIdVoting
operatorReward = deser._read_le_uint16() # operatorReward
scriptPayout = deser._read_varbytes() # scriptPayout
inputsHash = deser._read_nbytes(32) # inputsHash
platformNodeID = b''
platformP2PPort = 0
platformHTTPPort = 0
if version >= 2 and ntype == 1:
platformNodeID = deser._read_nbytes(20) # platformNodeID
platformP2PPort = deser._read_le_uint16() # platformP2PPort
platformHTTPPort = deser._read_le_uint16() # platformHTTPPort
payloadSig = deser._read_varbytes() # payloadSig
return DashProRegTx(
version=version,
type=ntype,
mode=mode,
collateralOutpoint=collateralOutpoint,
ipAddress=ipAddress,
port=port,
KeyIdOwner=KeyIdOwner,
PubKeyOperator=PubKeyOperator,
KeyIdVoting=KeyIdVoting,
operatorReward=operatorReward,
scriptPayout=scriptPayout,
inputsHash=inputsHash,
platformNodeID=platformNodeID,
platformP2PPort=platformP2PPort,
platformHTTPPort=platformHTTPPort,
payloadSig=payloadSig,
)
@dataclass(kw_only=True, slots=True)
class DashProUpServTx:
'''Class representing DIP3 ProUpServTx'''
version: int
type: int
proTxHash: bytes
ipAddress: bytes
port: int
scriptOperatorPayout: bytes
inputsHash: bytes
platformNodeID: bytes
platformP2PPort: int
platformHTTPPort: int
payloadSig: bytes
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.ipAddress) == 16
and len(self.inputsHash) == 32
and len(self.payloadSig) == 96)
res = pack_le_uint16(self.version) # version
if self.version >= 2:
res += pack_le_uint16(self.type) # type
res += (
self.proTxHash + # proTxHash
self.ipAddress + # ipAddress
pack_be_uint16(self.port) + # port
pack_varbytes(self.scriptOperatorPayout) + # scriptOperatorPayout
self.inputsHash) # inputsHash
if self.version >= 2:
res += (self.platformNodeID + # platformNodeID
pack_le_uint16(self.platformP2PPort) + # platformP2PPort
pack_le_uint16(self.platformHTTPPort)) # platformHTTPPort
res += self.payloadSig # payloadSig
return res
@classmethod
def read_tx_extra(cls, deser):
version = deser._read_le_uint16() # version
ntype = 0
if version >= 2:
ntype = deser._read_le_uint16() # type
proTxHash = deser._read_nbytes(32) # proTxHash
ipAddress = deser._read_nbytes(16) # ipAddress
port = deser._read_be_uint16() # port
scriptOperatorPayout = deser._read_varbytes() # scriptOperatorPayout
inputsHash = deser._read_nbytes(32) # inputsHash
platformNodeID = b''
platformP2PPort = 0
platformHTTPPort = 0
if version >= 2 and ntype == 1:
platformNodeID = deser._read_nbytes(20) # platformNodeID
platformP2PPort = deser._read_le_uint16() # platformP2PPort
platformHTTPPort = deser._read_le_uint16() # platformHTTPPort
payloadSig = deser._read_nbytes(96) # payloadSig
return DashProUpServTx(
version=version,
type=ntype,
proTxHash=proTxHash,
ipAddress=ipAddress,
port=port,
scriptOperatorPayout=scriptOperatorPayout,
inputsHash=inputsHash,
platformNodeID=platformNodeID,
platformP2PPort=platformP2PPort,
platformHTTPPort=platformHTTPPort,
payloadSig=payloadSig,
)
@dataclass(kw_only=True, slots=True)
class DashProUpRegTx:
'''Class representing DIP3 ProUpRegTx'''
version: int
proTxHash: bytes
mode: int
PubKeyOperator: bytes
KeyIdVoting: bytes
scriptPayout: bytes
inputsHash: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.PubKeyOperator) == 48
and len(self.KeyIdVoting) == 20
and len(self.inputsHash) == 32)
return (
pack_le_uint16(self.version) + # version
self.proTxHash + # proTxHash
pack_le_uint16(self.mode) + # mode
self.PubKeyOperator + # PubKeyOperator
self.KeyIdVoting + # KeyIdVoting
pack_varbytes(self.scriptPayout) + # scriptPayout
self.inputsHash + # inputsHash
pack_varbytes(self.payloadSig) # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return DashProUpRegTx(
version=deser._read_le_uint16(),
proTxHash=deser._read_nbytes(32),
mode=deser._read_le_uint16(),
PubKeyOperator=deser._read_nbytes(48),
KeyIdVoting=deser._read_nbytes(20),
scriptPayout=deser._read_varbytes(),
inputsHash=deser._read_nbytes(32),
payloadSig=deser._read_varbytes(),
)
@dataclass(kw_only=True, slots=True)
class DashProUpRevTx:
'''Class representing DIP3 ProUpRevTx'''
version: int
proTxHash: bytes
reason: int
inputsHash: bytes
payloadSig: bytes
def serialize(self):
assert (len(self.proTxHash) == 32
and len(self.inputsHash) == 32
and len(self.payloadSig) == 96)
return (
pack_le_uint16(self.version) + # version
self.proTxHash + # proTxHash
pack_le_uint16(self.reason) + # reason
self.inputsHash + # inputsHash
self.payloadSig # payloadSig
)
@classmethod
def read_tx_extra(cls, deser):
return DashProUpRevTx(
version=deser._read_le_uint16(),
proTxHash=deser._read_nbytes(32),
reason=deser._read_le_uint16(),
inputsHash=deser._read_nbytes(32),
payloadSig=deser._read_nbytes(96),
)
@dataclass(kw_only=True, slots=True)
class DashCbTx:
'''Class representing DIP4 coinbase special tx'''
version: int
height: int
merkleRootMNList: bytes
merkleRootQuorums: bytes
bestCLHeightDiff: int
bestCLSignature: bytes
assetLockedAmount: int
def serialize(self):
assert len(self.merkleRootMNList) == 32
res = (
pack_le_uint16(self.version) + # version
pack_le_uint32(self.height) + # height
self.merkleRootMNList # merkleRootMNList
)
if self.version > 1:
assert len(self.merkleRootQuorums) == 32
res += self.merkleRootQuorums # merkleRootQuorums
if self.version > 2:
res += pack_varint(self.bestCLHeightDiff)
res += self.bestCLSignature
res += pack_le_int64(self.assetLockedAmount)
return res
@classmethod
def read_tx_extra(cls, deser):
version = deser._read_le_uint16()
height = deser._read_le_uint32()
merkleRootMNList = deser._read_nbytes(32)
merkleRootQuorums = b''
bestCLHeightDiff = 0
bestCLSignature = b''
assetLockedAmount = 0
if version > 1:
merkleRootQuorums = deser._read_nbytes(32)
if version > 2:
bestCLHeightDiff = deser._read_varint()
bestCLSignature = deser._read_nbytes(96)
assetLockedAmount = deser._read_le_uint64()
return DashCbTx(
version=version,
height=height,
merkleRootMNList=merkleRootMNList,
merkleRootQuorums=merkleRootQuorums,
bestCLHeightDiff=bestCLHeightDiff,
bestCLSignature=bestCLSignature,
assetLockedAmount=assetLockedAmount,
)
# https://dash-docs.github.io/en/developer-reference#outpoint
@dataclass(kw_only=True, slots=True)
class TxOutPoint:
'''Class representing tx output outpoint'''
hash: bytes
index: int
def serialize(self):
assert len(self.hash) == 32
return (
self.hash + # hash
pack_le_uint32(self.index) # index
)
@classmethod
def read_outpoint(cls, deser):
return TxOutPoint(
hash=deser._read_nbytes(32),
index=deser._read_le_uint32(),
)
class DeserializerDash(Deserializer):
'''Deserializer for Dash DIP2 special tx types'''
# Supported Spec Tx types and corresponding classes mapping
PRO_REG_TX = 1
PRO_UP_SERV_TX = 2
PRO_UP_REG_TX = 3
PRO_UP_REV_TX = 4
CB_TX = 5
SPEC_TX_HANDLERS = {
PRO_REG_TX: DashProRegTx,
PRO_UP_SERV_TX: DashProUpServTx,
PRO_UP_REG_TX: DashProUpRegTx,
PRO_UP_REV_TX: DashProUpRevTx,
CB_TX: DashCbTx,
}
def _read_outpoint(self):
return TxOutPoint.read_outpoint(self)
def read_tx(self):
start = self.cursor
header = self._read_le_uint32()
tx_type = header >> 16 # DIP2 tx type
if tx_type:
version = header & 0x0000ffff
else:
version = header
if tx_type and version < 3:
version = header
tx_type = 0
inputs = self._read_inputs()
outputs = self._read_outputs()
locktime = self._read_le_uint32()
if tx_type:
extra_payload_size = self._read_varint()
end = self.cursor + extra_payload_size
spec_tx_class = DeserializerDash.SPEC_TX_HANDLERS.get(tx_type)
if spec_tx_class:
read_method = getattr(spec_tx_class, 'read_tx_extra', None)
extra_payload = read_method(self)
assert isinstance(extra_payload, spec_tx_class)
else:
extra_payload = self._read_nbytes(extra_payload_size)
assert self.cursor == end
else:
extra_payload = b''
txid = self.TX_HASH_FN(self.binary[start:self.cursor])
tx = DashTx(
version=version,
inputs=inputs,
outputs=outputs,
locktime=locktime,
tx_type=tx_type,
extra_payload=extra_payload,
txid_rev=txid,
wtxid_rev=txid,
)
return tx
src/electrumx/lib/util.py
+468
View File
@@ -0,0 +1,468 @@
# Copyright (c) 2016-2017, Neil Booth
#
# All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# and warranty status of this software.
'''Miscellaneous utility classes and functions.'''
from array import array
import asyncio
import inspect
from ipaddress import ip_address
import logging
import sys
from collections.abc import Container, Mapping
from struct import Struct
import time
from typing import Any, Optional
import aiorpcx
# Use orjson (faster 3rd party library) if available, fallback to stdlib
try:
import orjson
json_deserialize = orjson.loads
def json_serialize(obj: Any) -> str:
return orjson.dumps(obj).decode() # orjson.dumps() returns bytes
except ImportError:
import json
json_deserialize = json.loads
json_serialize = json.dumps
# Logging utilities
class ConnectionLogger(logging.LoggerAdapter):
'''Prepends a connection identifier to a logging message.'''
def process(self, msg, kwargs):
conn_id = self.extra.get('conn_id', 'unknown')
return f'[{conn_id}] {msg}', kwargs
class CompactFormatter(logging.Formatter):
'''Strips the module from the logger name to leave the class only.'''
def format(self, record):
record.name = record.name.rpartition('.')[-1]
return super().format(record)
def make_logger(name, *, handler, level):
'''Return the root ElectrumX logger.'''
logger = logging.getLogger(name)
logger.addHandler(handler)
logger.setLevel(level)
logger.propagate = False
return logger
def class_logger(path, classname):
'''Return a hierarchical logger for a class.'''
return logging.getLogger(path).getChild(classname)
def formatted_time(t, sep=' '):
'''Return a number of seconds as a string in days, hours, mins and
maybe secs.'''
t = int(t)
fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60))
parts = []
for fmt, n in fmts:
val = t // n
if parts or val:
parts.append(fmt.format(val))
t %= n
if len(parts) < 3:
parts.append(f'{t:02d}s')
return sep.join(parts)
def deep_getsizeof(obj):
"""Find the memory footprint of a Python object.
Based on code from code.tutsplus.com: http://goo.gl/fZ0DXK
This is a recursive function that drills down a Python object graph
like a dictionary holding nested dictionaries with lists of lists
and tuples and sets.
The sys.getsizeof function does a shallow size of only. It counts each
object inside a container as pointer only regardless of how big it
really is.
"""
ids = set()
def size(o):
if id(o) in ids:
return 0
r = sys.getsizeof(o)
ids.add(id(o))
if isinstance(o, (str, bytes, bytearray, array)):
return r
if isinstance(o, Mapping):
return r + sum(size(k) + size(v) for k, v in o.items())
if isinstance(o, Container):
return r + sum(size(x) for x in o)
return r
return size(obj)
def subclasses(base_class, strict=True):
'''Return a list of subclasses of base_class in its module.'''
def select(obj):
return (inspect.isclass(obj) and issubclass(obj, base_class) and
(not strict or obj != base_class))
pairs = inspect.getmembers(sys.modules[base_class.__module__], select)
return [pair[1] for pair in pairs]
def chunks(items, size):
'''Break up items, an iterable, into chunks of length size.'''
for i in range(0, len(items), size):
yield items[i: i + size]
def resolve_limit(limit):
if limit is None or limit < 0:
return -1
assert isinstance(limit, int)
return limit
def bytes_to_int(be_bytes):
'''Interprets a big-endian sequence of bytes as an integer'''
return int.from_bytes(be_bytes, 'big')
def int_to_bytes(value):
'''Converts an integer to a big-endian sequence of bytes'''
return value.to_bytes((value.bit_length() + 7) // 8, 'big')
def increment_byte_string(bs):
'''Return the lexicographically next byte string of the same length.
Return None if there is none (when the input is all 0xff bytes).'''
try:
return (int.from_bytes(bs, 'big') + 1).to_bytes(len(bs), 'big')
except OverflowError:
return None
class LogicalFile:
'''A logical binary file split across several separate files on disk.'''
def __init__(self, prefix, digits, file_size):
self.filename_fmt = f'{prefix}{{:0{digits:d}d}}'
self.file_size = file_size
def read(self, start, size=-1):
'''Read up to size bytes from the virtual file, starting at offset
start, and return them.
If size is -1 all bytes are read.'''
parts = []
while size != 0:
try:
with self.open_file(start, False) as f:
part = f.read(size)
if not part:
break
except FileNotFoundError:
break
parts.append(part)
start += len(part)
if size > 0:
size -= len(part)
return b''.join(parts)
def write(self, start, b):
'''Write the bytes-like object, b, to the underlying virtual file.'''
while b:
size = min(len(b), self.file_size - (start % self.file_size))
with self.open_file(start, True) as f:
f.write(b if size == len(b) else b[:size])
b = b[size:]
start += size
def open_file(self, start, create):
'''Open the virtual file and seek to start. Return a file handle.
Raise FileNotFoundError if the file does not exist and create
is False.
'''
file_num, offset = divmod(start, self.file_size)
filename = self.filename_fmt.format(file_num)
f = open_file(filename, create)
f.seek(offset)
return f
def open_file(filename, create=False):
'''Open the file name. Return its handle.'''
try:
return open(filename, 'rb+')
except FileNotFoundError:
if create:
return open(filename, 'wb+')
raise
def open_truncate(filename):
'''Open the file name. Return its handle.'''
return open(filename, 'wb+')
def address_string(address):
'''Return an address as a correctly formatted string.'''
host, port = address
try:
host = ip_address(host)
except ValueError:
pass
else:
if host.version == 6:
return f'[{host}]:{port:d}'
return f'{host}:{port:d}'
def protocol_tuple(s):
'''Converts a protocol version number, such as "1.0" to a tuple (1, 0).
If the version number is bad, (0, ) indicating version 0 is returned.'''
try:
return tuple(int(part) for part in s.split('.'))
except (TypeError, ValueError, AttributeError):
return (0, )
def version_string(ptuple):
'''Convert a version tuple such as (1, 2) to "1.2".
There is always at least one dot, so (1, ) becomes "1.0".'''
while len(ptuple) < 2:
ptuple += (0, )
return '.'.join(str(p) for p in ptuple)
def protocol_version(client_req, min_tuple, max_tuple):
'''Given a client's protocol version string, return a pair of
protocol tuples:
(negotiated version, client min request)
If the request is unsupported, the negotiated protocol tuple is
None.
'''
if client_req is None:
client_min = client_max = min_tuple
else:
if isinstance(client_req, list) and len(client_req) == 2:
client_min, client_max = client_req
else:
client_min = client_max = client_req
client_min = protocol_tuple(client_min)
client_max = protocol_tuple(client_max)
result = min(client_max, max_tuple)
if result < max(client_min, min_tuple) or result == (0, ):
result = None
return result, client_min
def is_hex_str(text: Any, *, allow_odd_len: bool = False) -> bool:
if not isinstance(text, str):
return False
if allow_odd_len and len(text) % 2 == 1:
return is_hex_str("0" + text, allow_odd_len=False)
try:
b = bytes.fromhex(text)
except Exception:
return False
# forbid whitespaces in text:
if len(text) != 2 * len(b):
return False
return True
struct_le_i = Struct('<i')
struct_le_q = Struct('<q')
struct_le_H = Struct('<H')
struct_le_I = Struct('<I')
struct_le_Q = Struct('<Q')
struct_be_H = Struct('>H')
struct_be_I = Struct('>I')
struct_be_Q = Struct('>Q')
structB = Struct('B')
unpack_le_int32_from = struct_le_i.unpack_from
unpack_le_int64_from = struct_le_q.unpack_from
unpack_le_uint16_from = struct_le_H.unpack_from
unpack_le_uint32_from = struct_le_I.unpack_from
unpack_le_uint64_from = struct_le_Q.unpack_from
unpack_be_uint16_from = struct_be_H.unpack_from
unpack_be_uint32_from = struct_be_I.unpack_from
unpack_le_uint32 = struct_le_I.unpack
unpack_le_uint64 = struct_le_Q.unpack
unpack_be_uint32 = struct_be_I.unpack
unpack_be_uint64 = struct_be_Q.unpack
pack_le_int32 = struct_le_i.pack
pack_le_int64 = struct_le_q.pack
pack_le_uint16 = struct_le_H.pack
pack_le_uint32 = struct_le_I.pack
pack_le_uint64 = struct_le_Q.pack
pack_be_uint16 = struct_be_H.pack
pack_be_uint32 = struct_be_I.pack
pack_be_uint64 = struct_be_Q.pack
pack_byte = structB.pack
hex_to_bytes = bytes.fromhex
def pack_varint(n):
if n < 253:
return pack_byte(n)
if n < 65536:
return pack_byte(253) + pack_le_uint16(n)
if n < 4294967296:
return pack_byte(254) + pack_le_uint32(n)
return pack_byte(255) + pack_le_uint64(n)
def pack_varbytes(data):
return pack_varint(len(data)) + data
class OldTaskGroup(aiorpcx.TaskGroup):
"""Automatically raises exceptions on join; as in aiorpcx prior to version 0.20"""
async def join(self):
if self._wait is all:
exc = False
try:
async for task in self:
if not task.cancelled():
task.result()
except BaseException: # including asyncio.CancelledError
exc = True
raise
finally:
if exc:
await self.cancel_remaining()
await super().join()
else:
await super().join()
if self.completed:
self.completed.result()
# We monkey-patch aiorpcx TimeoutAfter (used by timeout_after and ignore_after API),
# to fix a timing issue present in asyncio as a whole re timing out tasks.
# To see the issue we are trying to fix, consider example:
# async def outer_task():
# async with timeout_after(0.1):
# await inner_task()
# When the 0.1 sec timeout expires, inner_task will get cancelled by timeout_after
# (=internal cancellation).
# If around the same time (in terms of event loop iterations) another coroutine
# cancels outer_task (=external cancellation), there will be a race.
# Both cancellations work by propagating a CancelledError out to timeout_after, which then
# needs to decide (in TimeoutAfter.__aexit__) whether it's due to an internal or external cancel.
# AFAICT asyncio provides no reliable way of distinguishing between the two.
# This patch tries to always give priority to external cancellations.
# see https://github.com/kyuupichan/aiorpcX/issues/44
# see https://github.com/aio-libs/async-timeout/issues/229
# see https://bugs.python.org/issue42130 and https://bugs.python.org/issue45098
def _aiorpcx_monkeypatched_set_new_deadline(task, deadline):
def timeout_task():
task._orig_cancel()
task._timed_out = None if getattr(task, "_externally_cancelled", False) else deadline
def mycancel(*args, **kwargs):
task._orig_cancel(*args, **kwargs)
task._externally_cancelled = True
task._timed_out = None
if not hasattr(task, "_orig_cancel"):
task._orig_cancel = task.cancel
task.cancel = mycancel
task._deadline_handle = task._loop.call_at(deadline, timeout_task)
def _aiorpcx_monkeypatched_set_task_deadline(task, deadline):
ret = _aiorpcx_orig_set_task_deadline(task, deadline)
task._externally_cancelled = None
return ret
def _aiorpcx_monkeypatched_unset_task_deadline(task):
if hasattr(task, "_orig_cancel"):
task.cancel = task._orig_cancel
del task._orig_cancel
return _aiorpcx_orig_unset_task_deadline(task)
_aiorpcx_orig_set_task_deadline = aiorpcx.curio._set_task_deadline
_aiorpcx_orig_unset_task_deadline = aiorpcx.curio._unset_task_deadline
aiorpcx.curio._set_new_deadline = _aiorpcx_monkeypatched_set_new_deadline
aiorpcx.curio._set_task_deadline = _aiorpcx_monkeypatched_set_task_deadline
aiorpcx.curio._unset_task_deadline = _aiorpcx_monkeypatched_unset_task_deadline
def get_running_loop() -> Optional[asyncio.AbstractEventLoop]:
"""Returns the asyncio event loop that is *running in this thread*, if any."""
try:
return asyncio.get_running_loop()
except RuntimeError:
return None
class LogTimeTaken:
"""A simple context manager that logs how much time passed between enter and exit."""
def __init__(self, logger: logging.Logger, base_str: str, *, enabled: bool = True):
self.logger = logger
self.base_str = base_str
self.enabled = enabled
self.start_time = 0
def __enter__(self):
self.start_time = time.monotonic()
return None
def __exit__(self, exc_type, exc_val, exc_tb):
time_elapsed = time.monotonic() - self.start_time
if self.enabled:
self.logger.debug(f'{self.base_str} took {time_elapsed:.3f}s.')
src/electrumx/server/__init__.py
View File
Whitespace-only changes.
src/electrumx/server/block_processor.py
+1010
View File
File diff suppressed because it is too large. Load diff
src/electrumx/server/controller.py
+199
View File
@@ -0,0 +1,199 @@
# Copyright (c) 2016-2018, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
from asyncio import Event
from typing import Set, Dict, Tuple
from aiorpcx import _version as aiorpcx_version
import electrumx
from electrumx.lib.server_base import ServerBase
from electrumx.lib.util import version_string, OldTaskGroup
from electrumx.lib.tx import TxOutpoint
from electrumx.server.db import DB
from electrumx.server.mempool import MemPool, MemPoolAPI
from electrumx.server.session import SessionManager
class Notifications:
# hashX notifications come from two sources: new blocks and
# mempool refreshes.
#
# A user with a pending transaction is notified after the block it
# gets in is processed. Block processing can take an extended
# time, and the prefetcher might poll the daemon after the mempool
# code in any case. In such cases the transaction will not be in
# the mempool after the mempool refresh. We want to avoid
# notifying clients twice - for the mempool refresh and when the
# block is done. This object handles that logic by deferring
# notifications appropriately.
def __init__(self):
self._touched_hashxs_mp = {} # type: Dict[int, Set[bytes]]
self._touched_hashxs_bp = {} # type: Dict[int, Set[bytes]]
self._touched_outpoints_mp = {} # type: Dict[int, Set[TxOutpoint]]
self._touched_outpoints_bp = {} # type: Dict[int, Set[TxOutpoint]]
self._highest_block = -1
async def _maybe_notify(self):
th_mp, th_bp = self._touched_hashxs_mp, self._touched_hashxs_bp
# figure out block height
common_heights = set(th_mp).intersection(th_bp)
if common_heights:
height = max(common_heights)
elif th_mp and max(th_mp) == self._highest_block:
height = self._highest_block
else:
# Either we are processing a block and waiting for it to
# come in, or we have not yet had a mempool update for the
# new block height
return
# hashXs
touched_hashxs = th_mp.pop(height)
for old in [h for h in th_mp if h <= height]:
del th_mp[old]
for old in [h for h in th_bp if h <= height]:
touched_hashxs.update(th_bp.pop(old))
# outpoints
to_mp, to_bp = self._touched_outpoints_mp, self._touched_outpoints_bp
touched_outpoints = to_mp.pop(height)
for old in [h for h in to_mp if h <= height]:
del to_mp[old]
for old in [h for h in to_bp if h <= height]:
touched_outpoints.update(to_bp.pop(old))
await self.notify(
height=height,
touched_hashxs=touched_hashxs,
touched_outpoints=touched_outpoints,
)
async def notify(
self,
*,
touched_hashxs: Set[bytes],
touched_outpoints: Set[TxOutpoint],
height: int,
):
pass
async def start(self, height: int, notify_func):
self._highest_block = height
self.notify = notify_func
await self.notify(
height=height,
touched_hashxs=set(),
touched_outpoints=set(),
)
async def on_mempool(
self,
*,
touched_hashxs: Set[bytes],
touched_outpoints: Set[TxOutpoint],
height: int,
):
self._touched_hashxs_mp[height] = touched_hashxs
self._touched_outpoints_mp[height] = touched_outpoints
await self._maybe_notify()
async def on_block(
self,
*,
touched_hashxs: Set[bytes],
touched_outpoints: Set[TxOutpoint],
height: int,
):
self._touched_hashxs_bp[height] = touched_hashxs
self._touched_outpoints_bp[height] = touched_outpoints
self._highest_block = height
await self._maybe_notify()
class Controller(ServerBase):
'''Manages server initialisation and shutdown.
Servers are started once the mempool is synced after the block
processor first catches up with the daemon.
'''
async def serve(self, shutdown_event):
'''Start the RPC server and wait for the mempool to synchronize. Then
start serving external clients.
'''
if not (0, 25, 0) <= aiorpcx_version < (0, 26):
raise RuntimeError(
f'aiorpcX version {aiorpcx_version!r} does not match required: 0.25.0<=ver<0.26')
env = self.env
min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
self.logger.info(f'software version: {electrumx.version}')
self.logger.info(f'aiorpcX version: {version_string(aiorpcx_version)}')
self.logger.info(f'supported protocol versions: {min_str}-{max_str}')
self.logger.info(f'event loop policy: {env.loop_policy}')
self.logger.info(f'reorg limit is {env.reorg_limit:,d} blocks')
notifications = Notifications()
Daemon = env.coin.DAEMON
BlockProcessor = env.coin.BLOCK_PROCESSOR
async with Daemon(env.coin, env.daemon_url) as daemon:
db = DB(env)
bp = BlockProcessor(env, db, daemon, notifications)
# Set notifications up to implement the MemPoolAPI
def get_db_height():
return db.db_height
notifications.height = daemon.height
notifications.db_height = get_db_height
notifications.db_height_changed = db.db_flushed_event.wait
notifications.cached_height = daemon.cached_height
notifications.daemon_height_changed = daemon.height_changed.wait
notifications.mempool_txids_hum = daemon.mempool_txids_hum
notifications.raw_transactions = daemon.getrawtransactions
notifications.lookup_utxos = db.lookup_utxos
MemPoolAPI.register(Notifications)
mempool = MemPool(
env.coin, notifications,
refresh_secs=env.daemon_poll_interval_mempool_msec/1000,
)
session_mgr = SessionManager(
env=env,
db=db,
block_processor=bp,
daemon=daemon,
mempool=mempool,
shutdown_event=shutdown_event,
)
# Test daemon authentication, and also ensure it has a cached
# height. Do this before entering the task group.
await daemon.height()
# Check if daemon is recent enough
# note: we only check the first daemon
# (DAEMON_URL might be set to a list of daemons)
await daemon.check_daemon_version()
await daemon.check_daemon_indexes()
caught_up_event = Event()
mempool_event = Event()
async def wait_for_catchup():
await caught_up_event.wait()
await group.spawn(db.populate_header_merkle_cache())
await group.spawn(mempool.keep_synchronized(mempool_event))
try:
async with OldTaskGroup() as group:
await group.spawn(session_mgr.serve(notifications, mempool_event))
await group.spawn(bp.fetch_and_process_blocks(caught_up_event))
await group.spawn(wait_for_catchup())
except Exception as e:
self.logger.exception("taskgroup died.")
finally:
self.logger.info("taskgroup stopped.")
src/electrumx/server/daemon.py
+638
View File
@@ -0,0 +1,638 @@
# Copyright (c) 2016-2017, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Class for handling asynchronous connections to a blockchain
daemon.'''
import asyncio
import itertools
import time
from calendar import timegm
from struct import pack
from typing import TYPE_CHECKING, Type, Sequence, Any, Iterable, Optional
import aiohttp
from aiorpcx import JSONRPC
from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash
from electrumx.lib.tx import DeserializerDecred
from electrumx.lib.util import (class_logger, hex_to_bytes, json_deserialize,
json_serialize, pack_varint,
unpack_le_uint16_from)
if TYPE_CHECKING:
from electrumx.lib.coins import Coin
class DaemonError(Exception):
'''Raised when the daemon returns an error in its results.'''
class WarmingUpError(Exception):
'''Internal - when the daemon is warming up.'''
class ServiceRefusedError(Exception):
'''Internal - when the daemon doesn't provide a JSON response, only an HTTP error, for
some reason.'''
class DaemonParseError(Exception):
'''Internal - when the daemon returns an RPC_PARSE_ERROR.'''
class Daemon:
'''Handles connections to a daemon at the given URL.'''
RPC_IN_WARMUP = -28
RPC_PARSE_ERROR = -32700
id_counter = itertools.count()
def __init__(
self,
coin: Type['Coin'],
url,
*,
max_workqueue=10,
init_retry=0.25,
max_retry=4.0,
):
self.coin = coin
self.logger = class_logger(__name__, self.__class__.__name__)
self.url_index = None
self.urls = []
self.set_url(url)
# Limit concurrent RPC calls to this number.
# See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16
self.workqueue_semaphore = asyncio.Semaphore(value=max_workqueue)
self.init_retry = init_retry
self.max_retry = max_retry
self._height = None
self.height_changed = asyncio.Event()
self.available_rpcs = {}
self.session = None
self._networkinfo_cache = (None, 0)
self._networkinfo_lock = asyncio.Lock()
self._mempoolinfo_cache = (None, 0)
self._mempoolinfo_lock = asyncio.Lock()
async def __aenter__(self):
self.session = aiohttp.ClientSession(connector=self.connector())
return self
async def __aexit__(self, exc_type, exc_value, traceback):
await self.session.close()
self.session = None
def connector(self):
return None
async def check_daemon_version(self):
assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}"
if self.coin.MIN_REQUIRED_DAEMON_VERSION is None:
return
required_version = tuple(map(int, self.coin.MIN_REQUIRED_DAEMON_VERSION.split('.')))
network_info = await self.getnetworkinfo()
ni_version = network_info['version']
daemon_major = ni_version // 10_000
daemon_minor = (ni_version % 10_000) // 100
daemon_rev = ni_version % 100
daemon_version = (daemon_major, daemon_minor, daemon_rev)
if daemon_version < required_version:
raise RuntimeError(f"Bitcoin Core {daemon_version=} < {required_version=}.")
async def check_daemon_indexes(self):
assert self.session is not None and self.coin is not None, f"{self.session=}, {self.coin=}"
if not self.coin.REQUIRED_DAEMON_INDEXES:
return
index_info = await self.getindexinfo()
for req_index in self.coin.REQUIRED_DAEMON_INDEXES:
if req_index not in index_info:
raise RuntimeError(
f"bitcoind missing required index: {req_index}. "
f"You should set {req_index}=1 in your bitcoin.conf config file, and reindex.")
if not index_info[req_index]["synced"]:
# Should we raise? Not raising allows syncing a fresh bitcoind and e-x "in parallel".
self.logger.warning(f"bitcoind required index {req_index!r} is still syncing!")
def set_url(self, url: str) -> None:
'''Set the URLS to the given list, and switch to the first one.'''
urls = url.split(',')
urls = [self.coin.sanitize_url(url) for url in urls]
for n, url in enumerate(urls):
status = '' if n else ' (current)'
logged_url = self.logged_url(url)
self.logger.info(f'daemon #{n + 1} at {logged_url}{status}')
self.url_index = 0
self.urls = urls
def current_url(self) -> str:
'''Returns the current daemon URL.'''
return self.urls[self.url_index]
def logged_url(self, url=None):
'''The host and port part, for logging.'''
url = url or self.current_url()
return url[url.rindex('@') + 1:]
def failover(self):
'''Call to fail-over to the next daemon URL.
Returns False if there is only one, otherwise True.
'''
if len(self.urls) > 1:
self.url_index = (self.url_index + 1) % len(self.urls)
self.logger.info(f'failing over to {self.logged_url()}')
return True
return False
async def _send_data(self, data):
async with self.workqueue_semaphore:
async with self.session.post(self.current_url(), data=data) as resp:
kind = resp.headers.get('Content-Type', None)
if kind == 'application/json':
return await resp.json(loads=json_deserialize)
text = await resp.text()
text = text.strip() or resp.reason
raise ServiceRefusedError(text)
async def _send(self, payload, processor):
'''Send a payload to be converted to JSON.
Handles temporary connection issues. Daemon response errors
are raise through DaemonError.
'''
def log_error(error):
nonlocal last_error_log, retry
now = time.monotonic()
if now - last_error_log > 60:
last_error_log = now
self.logger.error(f'{error}. Retrying occasionally...')
if retry == self.max_retry and self.failover():
retry = 0
on_good_message = None
last_error_log = 0
data = json_serialize(payload)
retry = self.init_retry
while True:
try:
result = await self._send_data(data)
result = processor(result)
if on_good_message:
self.logger.info(on_good_message)
return result
except asyncio.TimeoutError:
log_error('timeout error')
except aiohttp.ServerDisconnectedError:
log_error('disconnected')
on_good_message = 'connection restored'
except ConnectionResetError:
log_error('connection reset')
on_good_message = 'connection restored'
except aiohttp.ClientConnectionError:
log_error('connection problem - check your daemon is running')
on_good_message = 'connection restored'
except aiohttp.ClientError as e:
log_error(f'daemon error: {e}')
on_good_message = 'running normally'
except ServiceRefusedError as e:
log_error(f'daemon service refused: {e}')
on_good_message = 'running normally'
except WarmingUpError:
log_error('starting up checking blocks')
on_good_message = 'running normally'
except DaemonParseError as e:
log_error(f"received RPC_PARSE_ERROR ({e=!r}). request was: {data=!r}")
on_good_message = 'running normally'
await asyncio.sleep(retry)
retry = max(min(self.max_retry, retry * 2), self.init_retry)
async def _send_single(self, method, params=None):
'''Send a single request to the daemon.'''
def processor(result):
err = result['error']
if not err:
return result['result']
if err.get('code') == self.RPC_IN_WARMUP:
raise WarmingUpError
if err.get('code') == self.RPC_PARSE_ERROR:
raise DaemonParseError(err)
raise DaemonError(err)
payload = {'method': method, 'id': next(self.id_counter)}
if params:
payload['params'] = params
return await self._send(payload, processor)
async def _send_vector(self, method, params_iterable, replace_errs=False):
'''Send several requests of the same method.
The result will be an array of the same length as params_iterable.
If replace_errs is true, any item with an error is returned as None,
otherwise an exception is raised.'''
def processor(result):
errs = [item['error'] for item in result if item['error']]
if any(err.get('code') == self.RPC_IN_WARMUP for err in errs):
raise WarmingUpError
if any(err.get('code') == self.RPC_PARSE_ERROR for err in errs):
raise DaemonParseError
if not errs or replace_errs:
return [item['result'] for item in result]
raise DaemonError(errs)
payload = [{'method': method, 'params': p, 'id': next(self.id_counter)}
for p in params_iterable]
if payload:
return await self._send(payload, processor)
return []
async def _is_rpc_available(self, method: str) -> bool:
'''Return whether given RPC method is available in the daemon.
Results are cached and the daemon will generally not be queried with
the same method more than once.'''
available = self.available_rpcs.get(method)
if available is None:
available = True
try:
await self._send_single(method)
except DaemonError as e:
err = e.args[0]
error_code = err.get("code")
available = error_code != JSONRPC.METHOD_NOT_FOUND
self.available_rpcs[method] = available
return available
async def block_hex_hashes(self, first: int, count: int) -> Sequence[str]:
'''Return the hex hashes (hum) of count block starting at height first.'''
params_iterable = ((h, ) for h in range(first, first + count))
return await self._send_vector('getblockhash', params_iterable)
async def deserialised_block(self, bhash_hum: str) -> dict:
'''Return the deserialised block with the given hex hash.'''
return await self._send_single('getblock', (bhash_hum, True))
async def raw_blocks(self, bhashes_hum: Sequence[str]) -> Sequence[bytes]:
'''Return the raw binary blocks with the given hex hashes.'''
params_iterable = ((h, False) for h in bhashes_hum)
blocks = await self._send_vector('getblock', params_iterable)
# Convert hex string to bytes
return [hex_to_bytes(block) for block in blocks]
async def mempool_txids_hum(self) -> Sequence[str]:
'''Update our record of the daemon's mempool hashes.'''
return await self._send_single('getrawmempool')
async def estimatefee(self, block_count, estimate_mode=None):
'''Return the fee estimate for the block count. Units are whole
currency units per KB, e.g. 0.00000995, or -1 if no estimate
is available.
'''
if estimate_mode:
args = (block_count, estimate_mode)
else:
args = (block_count, )
if await self._is_rpc_available('estimatesmartfee'):
estimate = await self._send_single('estimatesmartfee', args)
return estimate.get('feerate', -1)
return await self._send_single('estimatefee', args)
async def getnetworkinfo(self):
'''Return the result of the 'getnetworkinfo' RPC call.'''
async with self._networkinfo_lock:
cache_val, cache_time = self._networkinfo_cache
if time.time() - cache_time < 60: # seconds
return cache_val
val = await self._send_single('getnetworkinfo')
self._networkinfo_cache = (val, time.time())
return val
async def getmempoolinfo(self):
"""Return the result of the 'getmempoolinfo' RPC call."""
async with self._mempoolinfo_lock:
cache_val, cache_time = self._mempoolinfo_cache
if time.time() - cache_time < 60: # seconds
return cache_val
val = await self._send_single('getmempoolinfo')
self._mempoolinfo_cache = (val, time.time())
return val
async def getindexinfo(self):
"""Return the result of the 'getindexinfo' RPC call."""
# note: not cached as it's not exposed to client sessions
return await self._send_single('getindexinfo')
async def relayfee(self):
"""Same as getmempoolinfo['minrelaytxfee'].
The minimum fee required for a transaction to be relayed on by the daemon to the
bitcoin network. Doesn't guarantee mempool acceptance."""
network_info = await self.getnetworkinfo()
return network_info['relayfee']
async def mempool_info(self) -> dict[str, float]:
mempool_info = await self.getmempoolinfo()
return {
# Dynamic minimum fee rate in BTC/kvB for tx to be accepted.
# Is the maximum of minrelaytxfee and minimum mempool fee
'mempoolminfee': mempool_info['mempoolminfee'],
# Minimum relay fee for transactions in BTC/kvB, operator-configurable, static.
'minrelaytxfee': mempool_info['minrelaytxfee'],
# Minimum fee rate increment for mempool limiting or replacement in BTC/kvB,
# operator-configurable, static.
'incrementalrelayfee': mempool_info['incrementalrelayfee'],
}
async def getrawtransaction(self, txid_hum: str, verbose=False):
'''Return the serialized raw transaction with the given hash.'''
# Cast to int because some coin daemons are old and require it
return await self._send_single('getrawtransaction',
(txid_hum, int(verbose)))
async def getrawtransactions(self, txids_hum: Iterable[str], replace_errs=True) -> Sequence[bytes | None]:
'''Return the serialized raw transactions with the given hashes.
Replaces errors with None by default.'''
params_iterable = ((txid_hum, 0) for txid_hum in txids_hum)
txs = await self._send_vector('getrawtransaction', params_iterable,
replace_errs=replace_errs)
# Convert hex strings to bytes
return [hex_to_bytes(tx) if tx else None for tx in txs]
async def gettxspendingprevout(self, prev_txid_hum: str, txout_idx: int) -> dict[str, Any]:
"""Query the daemon to find (if any) the spender of given outpoint."""
outpoints = [{"txid": prev_txid_hum, "vout": txout_idx}, ]
options = {"mempool_only": False}
tx_items = await self._send_single('gettxspendingprevout', (outpoints, options))
return tx_items[0]
async def broadcast_transaction(self, raw_tx: str) -> str:
'''Broadcast a transaction to the network.'''
return await self._send_single('sendrawtransaction', (raw_tx, ))
async def broadcast_package(self, raw_txs: Sequence[str]):
"""Broadcast a package of transactions to the network using 'submitpackage'."""
return await self._send_single('submitpackage', (raw_txs, ))
async def testmempoolaccept(self, raw_txs: Sequence[str]):
"""Query the daemon to test mempool acceptance of txs,
without adding them to the mempool or broadcasting them.
"""
return await self._send_single('testmempoolaccept', (raw_txs, ))
async def height(self) -> int:
'''Query the daemon for its current height.'''
old_height = self._height
self._height = await self._send_single('getblockcount')
if old_height != self._height:
self.logger.debug(f"new height: {self._height}")
self.height_changed.set()
self.height_changed.clear()
return self._height
def cached_height(self) -> Optional[int]:
'''Return the cached daemon height.
If the daemon has not been queried yet this returns None.'''
return self._height
class DashDaemon(Daemon):
async def masternode_broadcast(self, params):
'''Broadcast a transaction to the network.'''
return await self._send_single('masternodebroadcast', params)
async def masternode_list(self, params):
'''Return the masternode status.'''
return await self._send_single('masternodelist', params)
async def protx(self, params):
'''Set of commands to execute ProTx related actions.'''
return await self._send_single('protx', params)
class FakeEstimateFeeDaemon(Daemon):
'''Daemon that simulates estimatefee and relayfee RPC calls. Coin that
wants to use this daemon must define ESTIMATE_FEE & RELAY_FEE'''
async def estimatefee(self, block_count):
'''Return the fee estimate for the given parameters.'''
return self.coin.ESTIMATE_FEE
async def relayfee(self):
"""The minimum fee required for a transaction to be relayed on by the server to the
bitcoin network. Doesn't guarantee mempool acceptance."""
return self.coin.RELAY_FEE
class LegacyRPCDaemon(Daemon):
'''Handles connections to a daemon at the given URL.
This class is useful for daemons that don't have the new 'getblock'
RPC call that returns the block in hex, the workaround is to manually
recreate the block bytes. The recreated block bytes may not be the exact
as in the underlying blockchain but it is good enough for our indexing
purposes.'''
async def raw_blocks(self, bhashes_hum):
'''Return the raw binary blocks with the given hex hashes.'''
params_iterable = ((h, ) for h in bhashes_hum)
block_info = await self._send_vector('getblock', params_iterable)
blocks = []
for i in block_info:
raw_block = await self.make_raw_block(i)
blocks.append(raw_block)
# Convert hex string to bytes
return blocks
async def make_raw_header(self, b):
pbh = b.get('previousblockhash')
if pbh is None:
pbh = '0' * 64
return b''.join([
pack('<L', b.get('version')),
hex_str_to_hash(pbh),
hex_str_to_hash(b.get('merkleroot')),
pack('<L', self.timestamp_safe(b['time'])),
pack('<L', int(b.get('bits'), 16)),
pack('<L', int(b.get('nonce')))
])
async def make_raw_block(self, b):
'''Construct a raw block'''
header = await self.make_raw_header(b)
transactions = []
if b.get('height') > 0:
transactions = await self.getrawtransactions(b.get('tx'), False)
raw_block = header
num_txs = len(transactions)
if num_txs > 0:
raw_block += pack_varint(num_txs)
raw_block += b''.join(transactions)
else:
raw_block += b'\x00'
return raw_block
def timestamp_safe(self, t):
if isinstance(t, int):
return t
return timegm(time.strptime(t, "%Y-%m-%d %H:%M:%S %Z"))
class FakeEstimateLegacyRPCDaemon(LegacyRPCDaemon, FakeEstimateFeeDaemon):
pass
class DecredDaemon(Daemon):
async def raw_blocks(self, bhashes_hum):
'''Return the raw binary blocks with the given hex hashes.'''
params_iterable = ((h, False) for h in bhashes_hum)
blocks = await self._send_vector('getblock', params_iterable)
raw_blocks = []
valid_tx_tree = {}
for block in blocks:
# Convert to bytes from hex
raw_block = hex_to_bytes(block)
raw_blocks.append(raw_block)
# Check if previous block is valid
prev = self.prev_hex_hash(raw_block)
votebits = unpack_le_uint16_from(raw_block[100:102])[0]
valid_tx_tree[prev] = self.is_valid_tx_tree(votebits)
processed_raw_blocks = []
for hash, raw_block in zip(bhashes_hum, raw_blocks):
if hash in valid_tx_tree:
is_valid = valid_tx_tree[hash]
else:
# Do something complicated to figure out if this block is valid
header = await self._send_single('getblockheader', (hash, ))
if 'nextblockhash' not in header:
raise DaemonError(f'Could not find next block for {hash}')
next_hash = header['nextblockhash']
next_header = await self._send_single('getblockheader',
(next_hash, ))
is_valid = self.is_valid_tx_tree(next_header['votebits'])
if is_valid:
processed_raw_blocks.append(raw_block)
else:
# If this block is invalid remove the normal transactions
self.logger.info(f'block {hash} is invalidated')
processed_raw_blocks.append(self.strip_tx_tree(raw_block))
return processed_raw_blocks
@staticmethod
def prev_hex_hash(raw_block):
return hash_to_hex_str(raw_block[4:36])
@staticmethod
def is_valid_tx_tree(votebits):
# Check if previous block was invalidated.
return bool(votebits & (1 << 0) != 0)
def strip_tx_tree(self, raw_block):
c = self.coin
assert issubclass(c.DESERIALIZER, DeserializerDecred)
d = c.DESERIALIZER(raw_block, start=c.BASIC_HEADER_SIZE)
d.read_tx_tree() # Skip normal transactions
# Create a fake block without any normal transactions
return raw_block[:c.BASIC_HEADER_SIZE] + b'\x00' + raw_block[d.cursor:]
async def height(self):
height = await super().height()
if height > 0:
# Lie about the daemon height as the current tip can be invalidated
height -= 1
self._height = height
return height
async def mempool_txids_hum(self):
mempool = await super().mempool_txids_hum()
# Add current tip transactions to the 'fake' mempool.
real_height = await self._send_single('getblockcount')
tip_hash = await self._send_single('getblockhash', (real_height,))
tip = await self.deserialised_block(tip_hash)
# Add normal transactions except coinbase
mempool += tip['tx'][1:]
# Add stake transactions if applicable
mempool += tip.get('stx', [])
return mempool
def connector(self):
# FIXME allow self signed certificates
return aiohttp.TCPConnector(verify_ssl=False)
class PreLegacyRPCDaemon(LegacyRPCDaemon):
'''Handles connections to a daemon at the given URL.
This class is useful for daemons that don't have the new 'getblock'
RPC call that returns the block in hex, and need the False parameter
for the getblock'''
async def deserialised_block(self, hex_hash):
'''Return the deserialised block with the given hex hash.'''
return await self._send_single('getblock', (hex_hash, False))
class SmartCashDaemon(Daemon):
async def masternode_broadcast(self, params):
'''Broadcast a smartnode to the network.'''
return await self._send_single('smartnodebroadcast', params)
async def masternode_list(self, params):
'''Return the smartnode status.'''
return await self._send_single('smartnodelist', params)
async def smartrewards(self, params):
'''Return smartrewards data.'''
return await self._send_single('smartrewards', params)
class ZcoinMtpDaemon(Daemon):
def strip_mtp_data(self, raw_block):
if self.coin.is_mtp(raw_block):
return \
raw_block[:self.coin.MTP_HEADER_DATA_START*2] + \
raw_block[self.coin.MTP_HEADER_DATA_END*2:]
return raw_block
async def raw_blocks(self, bhashes_hum):
'''Return the raw binary blocks with the given hex hashes.'''
params_iterable = ((h, False) for h in bhashes_hum)
blocks = await self._send_vector('getblock', params_iterable)
# Convert hex string to bytes
return [hex_to_bytes(self.strip_mtp_data(block)) for block in blocks]
async def masternode_broadcast(self, params):
'''Broadcast a transaction to the network.'''
return await self._send_single('znodebroadcast', params)
async def masternode_list(self, params):
'''Return the masternode status.'''
return await self._send_single('znodelist', params)
async def protx(self, params):
'''Set of commands to execute ProTx related actions.'''
return await self._send_single('protx', params)
src/electrumx/server/db.py
+820
View File
@@ -0,0 +1,820 @@
# Copyright (c) 2016-2020, Neil Booth
# Copyright (c) 2017, the ElectrumX authors
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Interface to the blockchain database.'''
from array import array
import ast
import asyncio
import os
import time
from bisect import bisect_right
from dataclasses import dataclass
from glob import glob
from typing import Dict, List, Sequence, Tuple, Optional, TYPE_CHECKING, Union, Callable, Iterable
from aiorpcx import run_in_thread, sleep
import electrumx.lib.util as util
from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN, hex_str_to_hash
from electrumx.lib.merkle import Merkle, MerkleCache
from electrumx.lib.util import (
formatted_time,
)
from electrumx.lib.tx import TXOSpendStatus
from electrumx.server.storage import db_class, Storage
from electrumx.server.history import (
History,
)
from electrumx.server.db_util import (
DBTooOldForMigrations,
pack_txnum, unpack_txnum, TXNUM_LEN,
pack_txoutidx, unpack_txoutidx, TXOUTIDX_LEN,
pack_satoshis_val, unpack_satoshis_val,
pack_block_height, unpack_block_height, BHEIGHT_LEN,
pack_dyn_header_offset, unpack_dyn_header_offset, DYN_HEADER_OFFSET_LEN,
)
if TYPE_CHECKING:
from electrumx.server.env import Env
@dataclass(slots=True, order=True)
class UTXO:
tx_num: int # index of tx in chain order
tx_pos: int # tx output idx
txid_rev: bytes | None
height: int # block height
value: int # in satoshis
@dataclass(slots=True, kw_only=True)
class FlushData:
height: int
tx_count: int
headers: list[bytes]
bhash_to_bheight: dict[bytes, int] # block_hash_rev -> block_height
block_txids_rev: list[bytes]
# The following are flushed to the UTXO DB if undo_infos is not None
undo_infos: dict[int, Sequence[bytes]] # height -> undo_info
adds: dict[bytes, bytes] # txid+out_idx -> hashX+tx_num+value_sats
deletes: list[bytes] # b'h' db keys, and b'u' db keys. order does not matter
tip: bytes
COMP_TXID_LEN = 4
class DB:
'''Simple wrapper of the backend database for querying.
Performs no DB update, though the DB will be cleaned on opening if
it was shutdown uncleanly.
'''
DB_VERSIONS = (11, )
DB_STATE_KEY = b'state'
utxo_db: Optional['Storage']
class DBError(Exception):
'''Raised on general DB errors generally indicating corruption.'''
def __init__(self, env: 'Env'):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.env = env
self.coin = env.coin
# Setup block header size handlers
if self.coin.STATIC_BLOCK_HEADERS:
self.header_offset = self.coin.static_header_offset # type: Callable[[int], int]
self.header_len = self.coin.static_header_len # type: Callable[[int], int]
else:
self.header_offset = self.dynamic_header_offset # type: Callable[[int], int]
self.header_len = self.dynamic_header_len # type: Callable[[int], int]
self.logger.info(f'switching current directory to {env.db_dir}')
os.chdir(env.db_dir)
self.db_class = db_class(self.env.db_engine)
self.history = History()
# Key: b'u' + address_hashX + txout_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer (in satoshis)
# "at address, at outpoint, there is a UTXO of value v"
# ---
# Key: b'h' + compressed_tx_hash + txout_idx + tx_num
# Value: hashX
# "some outpoint created a UTXO at address"
# ---
# Key: b'U' + block_height
# Value: byte-concat list of (hashX + tx_num + value_sats)
# "undo data: list of UTXOs spent at block height"
self.utxo_db = None
self.fs_height = -1
self.fs_tx_count = 0
self.db_height = -1
self.db_tx_count = 0
self.db_tip = None # type: Optional[bytes] # block_hash_rev
self.last_flush = time.time()
self.last_flush_tx_count = 0
self.wall_time = 0
self.first_sync = True
self.db_version = -1
self.db_flushed_event = asyncio.Event()
self.logger.info(f'using {self.env.db_engine} for DB backend')
# Header merkle cache
self.merkle = Merkle()
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes_rev)
# note: all on-disk and in-memory dbs are consistent with each other if BlockProcessor.tip == DB.tip.
# (the mempool is out-of-scope here)
# on-disk: raw block headers in chain order
self.headers_file = util.LogicalFile('meta/headers', 2, 16000000)
# on-disk: cumulative number of txs at the end of height N
self.tx_counts = None # type: Optional[array] # in-memory
self.tx_counts_file = util.LogicalFile('meta/txcounts', 2, 2000000)
# on-disk: 32 byte txids in chain order, allows (tx_num -> txid_rev) map
self.hashes_file = util.LogicalFile('meta/hashes', 4, 16000000)
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file = util.LogicalFile(
'meta/headers_offsets', 2, 16000000)
# in-memory: (block_hash_rev -> block_height) full map
self.bhash_to_bheight = None # type: Optional[dict[bytes, int]]
async def _read_tx_counts(self) -> None:
if self.tx_counts is not None:
return
# tx_counts[N] has the cumulative number of txs at the end of
# height N. So tx_counts[0] is 1 - the genesis coinbase
size = (self.db_height + 1) * 8
tx_counts = self.tx_counts_file.read(0, size)
assert len(tx_counts) == size
self.tx_counts = array('Q', tx_counts)
if self.tx_counts:
assert self.db_tx_count == self.tx_counts[-1]
else:
assert self.db_tx_count == 0
async def _open_dbs(self, *, for_sync: bool) -> None:
assert self.utxo_db is None
# First UTXO DB
def open_utxo_db():
self.utxo_db = self.db_class('utxo', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new database')
self.logger.info('creating metadata directory')
os.mkdir('meta')
with util.open_file('COIN', create=True) as f:
f.write(f'ElectrumX databases and metadata for '
f'{self.coin.NAME} {self.coin.NET}'.encode())
if not self.coin.STATIC_BLOCK_HEADERS:
self.headers_offsets_file.write(0, b'\0\0\0\0\0\0\0\0')
else:
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
self.read_utxo_state()
await run_in_thread(open_utxo_db)
# Then history DB
def open_hist_db():
self.history.open_db(
db_class=self.db_class,
for_sync=for_sync,
utxo_db_tx_count=self.db_tx_count,
)
self.clear_excess_undo_info()
await run_in_thread(open_hist_db)
# Now prepare in-memory structures.
# - Read TX counts (requires meta directory)
await self._read_tx_counts()
# - (block_hash_rev -> block_height) map
await self._prep_bhash_to_bheight_map()
async def open_for_sync(self) -> None:
'''Open the databases to sync to the daemon.
When syncing we want to reserve a lot of open files for the
synchronization. When serving clients we want the open files for
serving network connections.
'''
await self._open_dbs(for_sync=True)
async def open_for_serving(self) -> None:
'''Open the databases for serving. If they are already open they are
closed first.
'''
if self.utxo_db:
self.logger.info('closing DBs to re-open for serving')
self.utxo_db.close()
self.history.close_db()
self.utxo_db = None
await self._open_dbs(for_sync=False)
# Header merkle cache
async def populate_header_merkle_cache(self) -> None:
self.logger.info('populating header merkle cache...')
length = max(1, self.db_height - self.env.reorg_limit)
start = time.monotonic()
await self.header_mc.initialize(length)
elapsed = time.monotonic() - start
self.logger.info(f'header merkle cache populated in {elapsed:.1f}s')
async def header_branch_and_root(self, length: int, height: int) -> tuple[list[bytes], bytes]:
return await self.header_mc.branch_and_root(length, height)
# -- (block_hash -> block_height) map
async def _prep_bhash_to_bheight_map(self) -> None:
if self.bhash_to_bheight is not None:
return
self.logger.info('preparing bhash->bheight map...')
t0 = time.monotonic()
self.bhash_to_bheight = {}
count = self.db_height + 1
block_hashes = await self.fs_block_hashes_rev(0, count)
if len(block_hashes) != count:
raise Exception(
f"failed to prep bhash_to_bheight. "
f"wanted {count} bhashes, only got {len(block_hashes)}")
for bheight, bhash in enumerate(block_hashes):
self.bhash_to_bheight[bhash] = bheight
assert len(self.bhash_to_bheight) == self.db_height + 1
self.logger.info(f'bhash->bheight map populated in {time.monotonic() - t0:.1f}s')
def get_blockheight_from_blockhash(self, block_hash_hum: str) -> Optional[int]:
bhash_rev = hex_str_to_hash(block_hash_hum)
return self.bhash_to_bheight.get(bhash_rev, None)
# Flushing
def assert_flushed(self, flush_data: FlushData) -> None:
'''Asserts state is fully flushed.'''
assert flush_data.tx_count == self.fs_tx_count == self.db_tx_count
assert flush_data.height == self.fs_height == self.db_height
assert flush_data.tip == self.db_tip
assert not flush_data.headers
assert not flush_data.bhash_to_bheight
assert not flush_data.block_txids_rev
assert not flush_data.adds
assert not flush_data.deletes
assert not flush_data.undo_infos
self.history.assert_flushed()
def flush_dbs(
self,
*,
flush_data: FlushData,
flush_utxos: bool,
estimate_txs_remaining: Callable[[], float],
) -> None:
'''Flush out cached state. History is always flushed; UTXOs are
flushed if flush_utxos.'''
if flush_data.height == self.db_height:
assert flush_data.tip == self.db_tip, f"block_hash mismatch at {flush_data.height}"
self.assert_flushed(flush_data)
return
start_time = time.time()
prior_flush = self.last_flush
tx_delta = flush_data.tx_count - self.last_flush_tx_count
# 1. Flush to file system
self._flush_fs(flush_data)
# 2. Then hist_db
self._flush_history()
# 3. Then utxo_db
# Flush state last as it reads the wall time.
with self.utxo_db.write_batch() as batch:
if flush_utxos:
self._flush_utxo_db(batch, flush_data)
self._flush_state(batch)
# Update and put the wall time again - otherwise we drop the
# time it took to commit the batch
self._flush_state(self.utxo_db)
elapsed = self.last_flush - start_time
self.logger.info(f'flush took '
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
# Catch-up stats
if self.utxo_db.for_sync:
flush_interval = self.last_flush - prior_flush
tx_per_sec_gen = int(flush_data.tx_count / self.wall_time)
tx_per_sec_last = 1 + int(tx_delta / flush_interval)
eta = estimate_txs_remaining() / tx_per_sec_last
self.logger.info(f'tx/sec since genesis: {tx_per_sec_gen:,d}, '
f'since last flush: {tx_per_sec_last:,d}')
self.logger.info(f'sync time: {formatted_time(self.wall_time)} '
f'ETA: {formatted_time(eta)}')
def _flush_fs(self, flush_data: FlushData) -> None:
'''Write headers, tx counts and block tx hashes to the filesystem.
The first height to write is self.fs_height + 1. The FS
metadata is all append-only, so in a crash we just pick up
again from the height stored in the DB.
'''
prior_tx_count = (self.tx_counts[self.fs_height]
if self.fs_height >= 0 else 0)
assert len(flush_data.block_txids_rev) == len(flush_data.headers)
assert len(flush_data.bhash_to_bheight) == len(flush_data.headers)
assert flush_data.height == self.fs_height + len(flush_data.headers)
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
else 0)
assert len(self.tx_counts) == flush_data.height + 1
txids_rev = b''.join(flush_data.block_txids_rev)
flush_data.block_txids_rev.clear()
assert len(txids_rev) % 32 == 0
assert len(txids_rev) // 32 == flush_data.tx_count - prior_tx_count
# Write the headers, tx counts, and tx hashes
start_time = time.monotonic()
height_start = self.fs_height + 1
offset = self.header_offset(height_start)
self.headers_file.write(offset, b''.join(flush_data.headers))
self.fs_update_header_offsets(offset_start=offset, height_start=height_start, headers=flush_data.headers)
flush_data.headers.clear()
self.bhash_to_bheight.update(flush_data.bhash_to_bheight)
assert len(self.bhash_to_bheight) == flush_data.height + 1
flush_data.bhash_to_bheight.clear()
offset = height_start * self.tx_counts.itemsize
self.tx_counts_file.write(offset,
self.tx_counts[height_start:].tobytes())
offset = prior_tx_count * 32
self.hashes_file.write(offset, txids_rev)
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
if self.utxo_db.for_sync:
elapsed = time.monotonic() - start_time
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
def _flush_history(self) -> None:
self.history.flush()
def _flush_utxo_db(self, batch, flush_data: FlushData) -> None:
'''Flush the cached DB writes and UTXO set to the batch.'''
# Care is needed because the writes generated by flushing the
# UTXO state may have keys in common with our write cache or
# may be in the DB already.
start_time = time.monotonic()
add_count = len(flush_data.adds)
spend_count = len(flush_data.deletes) // 2
# Spends
batch_delete = batch.delete
for key in sorted(flush_data.deletes):
batch_delete(key)
flush_data.deletes.clear()
# New UTXOs
batch_put = batch.put
for key, value in flush_data.adds.items():
# key: txid+out_idx, value: hashX+tx_num+value_sats
hashX = value[:HASHX_LEN]
txout_idx = key[-TXOUTIDX_LEN:]
tx_num = value[HASHX_LEN: HASHX_LEN+TXNUM_LEN]
value_sats = value[-8:]
suffix = txout_idx + tx_num
batch_put(b'h' + key[:COMP_TXID_LEN] + suffix, hashX)
batch_put(b'u' + hashX + suffix, value_sats)
flush_data.adds.clear()
# Add new undo information
self.flush_undo_infos(batch_put, flush_data.undo_infos)
flush_data.undo_infos.clear()
# Delete old undo information
if not self.utxo_db.for_sync: # undo infos were only added if we are nearly caught up anyway
old_min_height = max(0, self.min_undo_height(self.db_height))
new_min_height = max(0, self.min_undo_height(flush_data.height))
for h in range(old_min_height, new_min_height):
key = self.undo_key(h)
batch_delete(key)
# Log stats
if self.utxo_db.for_sync:
block_count = flush_data.height - self.db_height
tx_count = flush_data.tx_count - self.db_tx_count
elapsed = time.monotonic() - start_time
self.logger.info(f'flushed {block_count:,d} blocks with '
f'{tx_count:,d} txs, {add_count:,d} UTXO adds, '
f'{spend_count:,d} spends in '
f'{elapsed:.1f}s, committing...')
self.db_height = flush_data.height
self.db_tx_count = flush_data.tx_count
self.db_tip = flush_data.tip
def _flush_state(self, batch):
'''Flush chain state to the batch.'''
now = time.time()
self.wall_time += now - self.last_flush
self.last_flush = now
self.last_flush_tx_count = self.fs_tx_count
self.write_utxo_state(batch)
def flush_backup(self, flush_data: FlushData, touched_hashxs: Iterable[bytes]) -> None:
'''Like flush_dbs() but when backing up. All UTXOs are flushed.'''
assert not flush_data.headers
assert not flush_data.block_txids_rev
assert flush_data.height < self.db_height
assert len(flush_data.bhash_to_bheight) == (self.db_height - flush_data.height)
self.history.assert_flushed()
start_time = time.time()
tx_delta = flush_data.tx_count - self.last_flush_tx_count
# 1. (fake-)Flush to file system: this just updates pointers
self._backup_fs(flush_data)
# Crucially, we flush utxo_db and hist_db in the reverse order compared to when advancing,
# to maintain the invariant that hist_db_tx_count >= utxo_db_tx_count even if we crash.
# 2. Then utxo_db
with self.utxo_db.write_batch() as batch:
self._flush_utxo_db(batch, flush_data)
# Flush state last as it reads the wall time.
self._flush_state(batch)
# 3. Then hist_db
self.history.backup(
hashXs=touched_hashxs,
tx_count=flush_data.tx_count,
)
# Update and put the wall time again - otherwise we drop the
# time it took to commit the batch
self._flush_state(self.utxo_db)
elapsed = self.last_flush - start_time
self.logger.info(f'backup flush took '
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
def fs_update_header_offsets(self, *, offset_start: int, height_start: int, headers: Sequence[bytes]) -> None:
if self.coin.STATIC_BLOCK_HEADERS:
return
offset = offset_start
offsets = []
for h in headers:
offset += len(h)
offsets.append(pack_dyn_header_offset(offset))
# For each header we get the offset of the next header, hence we
# start writing from the next height
pos = (height_start + 1) * DYN_HEADER_OFFSET_LEN
self.headers_offsets_file.write(pos, b''.join(offsets))
def dynamic_header_offset(self, height: int) -> int:
assert not self.coin.STATIC_BLOCK_HEADERS
offset = unpack_dyn_header_offset(self.headers_offsets_file.read(height * DYN_HEADER_OFFSET_LEN, DYN_HEADER_OFFSET_LEN))
return offset
def dynamic_header_len(self, height: int) -> int:
return self.dynamic_header_offset(height + 1)\
- self.dynamic_header_offset(height)
def _backup_fs(self, flush_data: FlushData) -> None:
'''Back up during a reorg. This just updates our pointers.'''
self.fs_height = flush_data.height
self.fs_tx_count = flush_data.tx_count
for bhash, bheight in flush_data.bhash_to_bheight.items():
assert bheight == self.bhash_to_bheight.pop(bhash)
flush_data.bhash_to_bheight.clear()
assert len(self.bhash_to_bheight) == flush_data.height + 1
# Truncate header_mc: header count is 1 more than the height.
self.header_mc.truncate(flush_data.height + 1)
async def raw_header(self, height: int) -> bytes:
'''Return the binary header at the given height.'''
header, n = await self.read_headers(height, 1)
if n != 1:
raise IndexError(f'height {height:,d} out of range')
return header
async def read_headers(self, start_height: int, count: int) -> Tuple[bytes, int]:
'''Requires start_height >= 0, count >= 0. Reads as many headers as
are available starting at start_height up to count. This
would be zero if start_height is beyond self.db_height, for
example.
Returns a (binary, n) pair where binary is the concatenated
binary headers, and n is the count of headers returned.
'''
if start_height < 0 or count < 0:
raise self.DBError(f'{count:,d} headers starting at '
f'{start_height:,d} not on disk')
def read_headers():
# Read some from disk
disk_count = max(0, min(count, self.db_height + 1 - start_height))
if disk_count:
offset = self.header_offset(start_height)
size = self.header_offset(start_height + disk_count) - offset
return self.headers_file.read(offset, size), disk_count
return b'', 0
return await run_in_thread(read_headers)
def fs_txid_rev(self, tx_num: int) -> Tuple[Optional[bytes], int]:
'''Return a pair (txid_rev, tx_height) for the given tx number.
If the tx_height is not on disk, returns (None, tx_height).'''
tx_height = bisect_right(self.tx_counts, tx_num)
if tx_height > self.db_height:
txid_rev = None
else:
txid_rev = self.hashes_file.read(tx_num * 32, 32)
return txid_rev, tx_height
def fs_txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]:
'''Return a list of txids_rev at given block height,
in the same order as in the block.
'''
if block_height > self.db_height:
raise self.DBError(f'block {block_height:,d} not on disk (>{self.db_height:,d})')
assert block_height >= 0
if block_height > 0:
first_tx_num = self.tx_counts[block_height - 1]
else:
first_tx_num = 0
num_txs_in_block = self.tx_counts[block_height] - first_tx_num
txids_rev = self.hashes_file.read(first_tx_num * 32, num_txs_in_block * 32)
assert num_txs_in_block == len(txids_rev) // 32
return [txids_rev[idx * 32: (idx+1) * 32] for idx in range(num_txs_in_block)]
async def txids_rev_at_blockheight(self, block_height: int) -> Sequence[bytes]:
return await run_in_thread(self.fs_txids_rev_at_blockheight, block_height)
async def fs_block_hashes_rev(self, height: int, count: int) -> Sequence[bytes]:
headers_concat, headers_count = await self.read_headers(height, count)
if headers_count != count:
raise self.DBError(f'only got {headers_count:,d} headers starting '
f'at {height:,d}, not {count:,d}')
def hash_headers():
offset = 0
headers = []
for n in range(count):
hlen = self.header_len(height + n)
headers.append(headers_concat[offset:offset + hlen])
offset += hlen
return [self.coin.header_hash_rev(header) for header in headers]
bhashes = (await run_in_thread(hash_headers)) if count > 100 else hash_headers()
return bhashes
async def limited_history(self, hashX: bytes, *, limit: int = 1000) -> Sequence[tuple[bytes, int]]:
'''Return an unpruned, sorted list of (txid_rev, height) tuples of
confirmed transactions that touched the address, earliest in
the blockchain first. Includes both spending and receiving
transactions. By default returns at most 1000 entries. Set
limit to None to get them all.
'''
def read_history():
tx_nums = list(self.history.get_txnums(hashX, limit))
fs_txid_rev = self.fs_txid_rev
return [fs_txid_rev(tx_num) for tx_num in tx_nums]
while True:
history = await run_in_thread(read_history)
if all(txid_rev is not None for txid_rev, height in history):
return history
self.logger.warning(f'limited_history: tx hash '
f'not found (reorg?), retrying...')
await sleep(0.25)
# -- Undo information
def min_undo_height(self, max_height: int) -> int:
'''Returns a height from which we should store undo info.'''
return max_height - self.env.reorg_limit + 1
def undo_key(self, height: int) -> bytes:
'''DB key for undo information at the given height.'''
return b'U' + pack_block_height(height)
def read_undo_info(self, height: int) -> bytes:
'''Read undo information from a file for the current height.'''
return self.utxo_db.get(self.undo_key(height))
def flush_undo_infos(
self, batch_put, undo_infos: dict[int, Sequence[bytes]],
) -> None:
'''undo_infos is a (height -> undo_info) map.'''
for height, undo_info in undo_infos.items():
batch_put(self.undo_key(height), b''.join(undo_info))
def raw_block_prefix(self) -> str:
return 'meta/block'
def raw_block_path(self, height: int) -> str:
return f'{self.raw_block_prefix()}{height:d}'
def read_raw_block(self, height: int) -> bytes:
'''Returns a raw block read from disk. Raises FileNotFoundError
if the block isn't on-disk.'''
with util.open_file(self.raw_block_path(height)) as f:
return f.read(-1)
def write_raw_block(self, block: bytes, height: int) -> None:
'''Write a raw block to disk.'''
with util.open_truncate(self.raw_block_path(height)) as f:
f.write(block)
# Delete old blocks to prevent them accumulating
del_height = self.min_undo_height(height) - 1
if del_height >= 0:
try:
os.remove(self.raw_block_path(del_height))
except FileNotFoundError:
pass
def clear_excess_undo_info(self) -> None:
'''Clear excess undo info. Only most recent N are kept.'''
# delete aged undo_infos from utxo db
# note: this is just a fallback cleanup path - normally already deleted by _flush_utxo_db()
min_height = self.min_undo_height(self.db_height)
keys = []
for key, uhist in self.utxo_db.iterator(prefix=b'U'):
height = unpack_block_height(key[-BHEIGHT_LEN:])
if height >= min_height:
break # can break as block_height is encoded as big endian
keys.append(key)
if keys:
with self.utxo_db.write_batch() as batch:
for key in keys:
batch.delete(key)
self.logger.info(f'deleted {len(keys):,d} stale undo entries')
# delete old block files
# note: this is just a fallback cleanup path - normally already deleted by write_raw_block()
prefix = self.raw_block_prefix()
paths = [path for path in glob(f'{prefix}[0-9]*')
if len(path) > len(prefix)
and int(path[len(prefix):]) < min_height]
if paths:
for path in paths:
try:
os.remove(path)
except FileNotFoundError:
pass
self.logger.info(f'deleted {len(paths):,d} stale block files')
# -- UTXO database
def read_utxo_state(self) -> None:
state = self.utxo_db.get(self.DB_STATE_KEY)
if not state:
self.db_height = -1
self.db_tx_count = 0
self.db_tip = b'\0' * 32
self.db_version = max(self.DB_VERSIONS)
self.wall_time = 0
self.first_sync = True
else:
state = ast.literal_eval(state.decode())
if not isinstance(state, dict):
raise self.DBError('failed reading state from DB')
self.db_version = state['db_version']
if self.db_version not in self.DB_VERSIONS:
raise DBTooOldForMigrations(
db_name="UTXO", db_version=self.db_version, supported_versions=self.DB_VERSIONS)
genesis_hash = state['genesis']
if genesis_hash != self.coin.GENESIS_HASH:
raise self.DBError(f'DB genesis hash {genesis_hash} does not '
f'match coin {self.coin.GENESIS_HASH}')
self.db_height = state['height']
self.db_tx_count = state['tx_count']
self.db_tip = state['tip']
self.wall_time = state['wall_time']
self.first_sync = state['first_sync']
# These are our state as we move ahead of DB state
self.fs_height = self.db_height
self.fs_tx_count = self.db_tx_count
self.last_flush_tx_count = self.fs_tx_count
# Upgrade DB
if self.db_version != max(self.DB_VERSIONS):
raise Exception("missing db upgrade") # call future upgrade logic here
# Log some stats
self.logger.info(f'UTXO DB version: {self.db_version:d}')
self.logger.info(f'coin: {self.coin.NAME}')
self.logger.info(f'network: {self.coin.NET}')
self.logger.info(f'height: {self.db_height:,d}')
self.logger.info(f'tip: {hash_to_hex_str(self.db_tip)}')
self.logger.info(f'tx count: {self.db_tx_count:,d}')
if self.utxo_db.for_sync:
self.logger.info(f'flushing DB cache at {self.env.cache_MB:,d} MB')
if self.first_sync:
self.logger.info(
f'sync time so far: {util.formatted_time(self.wall_time)}'
)
def write_utxo_state(self, batch) -> None:
'''Write (UTXO) state to the batch.'''
state = {
'genesis': self.coin.GENESIS_HASH,
'height': self.db_height,
'tx_count': self.db_tx_count,
'tip': self.db_tip,
'wall_time': self.wall_time,
'first_sync': self.first_sync,
'db_version': self.db_version,
}
batch.put(self.DB_STATE_KEY, repr(state).encode())
async def all_utxos(self, hashX: bytes) -> Sequence[UTXO]:
'''Return all UTXOs for an address sorted in no particular order.'''
def read_utxos():
utxos = [] # type: list[UTXO]
utxos_append = utxos.append
# Key: b'u' + address_hashX + txout_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
prefix = b'u' + hashX
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
txout_idx = unpack_txoutidx(db_key[-TXNUM_LEN-TXOUTIDX_LEN:-TXNUM_LEN])
tx_num = unpack_txnum(db_key[-TXNUM_LEN:])
value = unpack_satoshis_val(db_value)
txid_rev, height = self.fs_txid_rev(tx_num)
utxos_append(UTXO(tx_num, txout_idx, txid_rev, height, value))
return utxos
while True:
utxos = await run_in_thread(read_utxos)
if all(utxo.txid_rev is not None for utxo in utxos):
return utxos
self.logger.warning(f'all_utxos: tx hash not '
f'found (reorg?), retrying...')
await sleep(0.25)
async def lookup_utxos(self, prevouts: Sequence[tuple[bytes, int]]) -> Sequence[Optional[tuple[bytes, int]]]:
'''For each prevout, lookup it up in the DB and return a (hashX,
value) pair or None if not found.
Used by the mempool code.
'''
def lookup_hashXs() -> Sequence[tuple[Optional[bytes], Optional[bytes]]]:
'''Return (hashX, suffix) pairs, or None if not found,
for each prevout.
'''
def lookup_hashX(txid_rev: bytes, tx_idx: int) -> tuple[Optional[bytes], Optional[bytes]]:
idx_packed = pack_txoutidx(tx_idx)
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
# Value: hashX
prefix = b'h' + txid_rev[:COMP_TXID_LEN] + idx_packed
# Find which entry, if any, the TX_HASH matches.
for db_key, hashX in self.utxo_db.iterator(prefix=prefix):
tx_num_packed = db_key[-TXNUM_LEN:]
tx_num = unpack_txnum(tx_num_packed)
hash, _height = self.fs_txid_rev(tx_num)
if hash == txid_rev:
return hashX, idx_packed + tx_num_packed
return None, None
return [lookup_hashX(*prevout) for prevout in prevouts]
def lookup_utxos(
hashX_pairs: Sequence[tuple[Optional[bytes], Optional[bytes]]],
) -> Sequence[Optional[tuple[bytes, int]]]:
def lookup_utxo(hashX: bytes, suffix: bytes) -> Optional[tuple[bytes, int]]:
if not hashX:
# This can happen when the daemon is a block ahead
# of us and has mempool txs spending outputs from
# that new block
return None
# Key: b'u' + address_hashX + tx_idx + tx_num
# Value: the UTXO value as a 64-bit unsigned integer
key = b'u' + hashX + suffix
db_value = self.utxo_db.get(key)
if not db_value:
# This can happen if the DB was updated between
# getting the hashXs and getting the UTXOs
return None
value = unpack_satoshis_val(db_value)
return hashX, value
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
hashX_pairs = await run_in_thread(lookup_hashXs)
return await run_in_thread(lookup_utxos, hashX_pairs)
src/electrumx/server/db_util.py
+88
View File
@@ -0,0 +1,88 @@
# Copyright (C) 2026 The Electrum developers
# Distributed under the MIT software license, see the accompanying
# file LICENCE or http://www.opensource.org/licenses/mit-license.php
from typing import Sequence
from electrumx.lib.util import (
pack_be_uint32, unpack_be_uint32,
pack_be_uint64, unpack_be_uint64,
)
# txnum
TXNUM_LEN = 5
TXNUM_PADDING = bytes(8 - TXNUM_LEN)
def unpack_txnum(tx_numb: bytes) -> int:
return unpack_be_uint64(TXNUM_PADDING + tx_numb)[0]
def pack_txnum(tx_num: int) -> bytes:
return pack_be_uint64(tx_num)[-TXNUM_LEN:]
assert pack_txnum(0x123) == bytes.fromhex("0000000123")
assert unpack_txnum(pack_txnum(0x123)) == 0x123
# txout_idx
TXOUTIDX_LEN = 3
TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN)
def unpack_txoutidx(txout_idx: bytes) -> int:
return unpack_be_uint32(TXOUTIDX_PADDING + txout_idx)[0]
def pack_txoutidx(txout_idx: int) -> bytes:
return pack_be_uint32(txout_idx)[-TXOUTIDX_LEN:]
assert pack_txoutidx(0x123) == bytes.fromhex("000123")
assert unpack_txoutidx(pack_txoutidx(0x123)) == 0x123
# sats
def unpack_satoshis_val(sats: bytes) -> int:
return unpack_be_uint64(sats)[0]
def pack_satoshis_val(sats: int) -> bytes:
return pack_be_uint64(sats)
# block height
BHEIGHT_LEN = 4
def unpack_block_height(bheight: bytes) -> int:
return unpack_be_uint32(bheight)[0]
def pack_block_height(bheight: int) -> bytes:
return pack_be_uint32(bheight)
# dynamic block header offsets
DYN_HEADER_OFFSET_LEN = 8
def unpack_dyn_header_offset(offset: bytes) -> int:
return unpack_be_uint64(offset)[0]
def pack_dyn_header_offset(offset: int) -> bytes:
return pack_be_uint64(offset)
class DBTooOldForMigrations(RuntimeError):
def __init__(self, *, db_name: str, db_version: int, supported_versions: Sequence[int]):
cmd = 'rm -rf DB_DIRECTORY/{hist,meta,utxo}'
super().__init__(
f'Your {db_name} DB version is {db_version} but this software only handles versions {supported_versions}. '
f'Manually delete your database (e.g. `{cmd}`, and start again. '
f'Then, your DB will be rebuilt from genesis, likely taking several hours. '
f"If you don't have time for this now, you can temporarily downgrade the software."
)
src/electrumx/server/env.py
+245
View File
@@ -0,0 +1,245 @@
# Copyright (c) 2016, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Class for handling environment configuration and defaults.'''
import re
from ipaddress import IPv4Address, IPv6Address
from typing import Type, Sequence
from aiorpcx import Service, ServicePart
import electrumx
from electrumx.lib.coins import Coin
from electrumx.lib.env_base import EnvBase
class ServiceError(Exception):
pass
class Env(EnvBase):
'''Wraps environment configuration. Optionally, accepts a Coin class
as first argument to have ElectrumX serve custom coins not part of
the standard distribution.
'''
# Peer discovery
PD_OFF, PD_SELF, PD_ON = ('OFF', 'SELF', 'ON')
SSL_PROTOCOLS = {'ssl', 'wss'}
KNOWN_PROTOCOLS = {'ssl', 'tcp', 'ws', 'wss', 'rpc'}
coin: Type[Coin]
def __init__(self, coin=None):
super().__init__()
self.obsolete(["MAX_SUBSCRIPTIONS", "MAX_SUBS", "MAX_SESSION_SUBS", "BANDWIDTH_LIMIT",
"HOST", "TCP_PORT", "SSL_PORT", "RPC_HOST", "RPC_PORT", "REPORT_HOST",
"REPORT_TCP_PORT", "REPORT_SSL_PORT", "REPORT_HOST_TOR",
"REPORT_TCP_PORT_TOR", "REPORT_SSL_PORT_TOR"])
# Core items
self.db_dir = self.required('DB_DIRECTORY')
self.db_engine = self.db_engine_enum()
self.daemon_url = self.required('DAEMON_URL')
if coin is not None:
assert issubclass(coin, Coin)
self.coin = coin
else:
coin_name = self.default('COIN', 'Bitcoin').strip()
network = self.default('NET', 'mainnet').strip()
self.coin = Coin.lookup_coin_class(coin_name, network)
# Peer discovery
self.peer_discovery = self.peer_discovery_enum()
self.peer_announce = self.boolean('PEER_ANNOUNCE', True)
self.force_proxy = self.boolean('FORCE_PROXY', False)
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
# Misc
self.banner_file = self.default('BANNER_FILE', None)
self.tor_banner_file = self.default('TOR_BANNER_FILE',
self.banner_file)
self.anon_logs = self.boolean('ANON_LOGS', False)
self.log_sessions = self.integer('LOG_SESSIONS', 3600)
self.log_level = self.default('LOG_LEVEL', 'info').upper()
self.donation_address = self.default('DONATION_ADDRESS', '')
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
self.cache_MB = self.integer('CACHE_MB', 1200)
self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
self.daemon_poll_interval_blocks_msec = self.integer('DAEMON_POLL_INTERVAL_BLOCKS', 5000)
self.daemon_poll_interval_mempool_msec = self.integer('DAEMON_POLL_INTERVAL_MEMPOOL', 5000)
# Server limits to help prevent DoS
self.max_send = self.integer('MAX_SEND', self.coin.DEFAULT_MAX_SEND)
self.max_recv = self.integer('MAX_RECV', self.coin.DEFAULT_MAX_RECV)
self._try_raising_rlimit_nofile() # do this before setting max_sessions
self.max_sessions = self.sane_max_sessions()
self.cost_soft_limit = self.integer('COST_SOFT_LIMIT', 1000)
self.cost_hard_limit = self.integer('COST_HARD_LIMIT', 10000)
self.bw_unit_cost = self.integer('BANDWIDTH_UNIT_COST', 5000)
self.initial_concurrent = self.integer('INITIAL_CONCURRENT', 10)
self.request_sleep = self.integer('REQUEST_SLEEP', 2500)
self.request_timeout = self.integer('REQUEST_TIMEOUT', 30)
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
self.session_group_by_subnet_ipv4 = self.integer('SESSION_GROUP_BY_SUBNET_IPV4', 24)
self.session_group_by_subnet_ipv6 = self.integer('SESSION_GROUP_BY_SUBNET_IPV6', 48)
self._check_and_fix_cost_limits()
# Services last - uses some env vars above
self.services = self.services_to_run()
if {service.protocol for service in self.services}.intersection(self.SSL_PROTOCOLS):
self.ssl_certfile = self.required('SSL_CERTFILE')
self.ssl_keyfile = self.required('SSL_KEYFILE')
self.report_services = self.services_to_report()
def _try_raising_rlimit_nofile(self) -> None:
"""Try raising the max num open file soft limit to the hard limit."""
try:
import resource
except ImportError:
return # No resource module on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft >= hard:
return
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
except (ValueError, OSError):
self.logger.warning(
f"error raising RLIMIT_NOFILE (ulimit -n) soft limit. Stuck with {soft}. "
f"You must ensure that electrumx has a large open file limit. See HOWTO in docs.")
else:
self.logger.info(
f"successfully raised RLIMIT_NOFILE (ulimit -n) soft limit from {soft} to {hard}")
def sane_max_sessions(self):
'''Return the maximum number of sessions to permit. Normally this
is MAX_SESSIONS. However, to prevent open file exhaustion, adjust
downwards if running with a small open file rlimit.'''
env_value = self.integer('MAX_SESSIONS', 1000)
# No resource module on Windows
try:
import resource
nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
# We give the DB 250 files; allow ElectrumX 100 for itself
value = max(0, min(env_value, nofile_limit - 350))
if value < env_value:
self.logger.warning(
f'lowered maximum sessions from {env_value:,d} to '
f'{value:,d} because your open file limit is '
f'{nofile_limit:,d}'
)
except ImportError:
value = 512 # that is what returned by stdio's _getmaxstdio()
return value
def _check_and_fix_cost_limits(self):
if self.cost_hard_limit < self.cost_soft_limit:
raise self.Error(f"COST_HARD_LIMIT must be >= COST_SOFT_LIMIT. "
f"got (COST_HARD_LIMIT={self.cost_hard_limit} "
f"and COST_SOFT_LIMIT={self.cost_soft_limit})")
# hard limit should be strictly higher than soft limit (unless both are 0)
if self.cost_hard_limit == self.cost_soft_limit and self.cost_soft_limit > 0:
self.logger.info("found COST_HARD_LIMIT == COST_SOFT_LIMIT. "
"bumping COST_HARD_LIMIT by 1.")
self.cost_hard_limit = self.cost_soft_limit + 1
def _parse_services(self, services_str, default_func) -> Sequence[Service]:
result = []
for service_str in services_str.split(','):
if not service_str:
continue
try:
service = Service.from_string(service_str, default_func=default_func)
except Exception as e:
raise ServiceError(f'"{service_str}" invalid: {e}') from None
if service.protocol not in self.KNOWN_PROTOCOLS:
raise ServiceError(f'"{service_str}" invalid: unknown protocol')
result.append(service)
# Find duplicate addresses
service_map = {service.address: [] for service in result}
for service in result:
service_map[service.address].append(service)
for address, services in service_map.items():
if len(services) > 1:
raise ServiceError(f'address {address} has multiple services')
return result
def services_to_run(self) -> Sequence[Service]:
def default_part(protocol, part):
return default_services.get(protocol, {}).get(part)
default_services = {protocol: {ServicePart.HOST: 'all_interfaces'}
for protocol in self.KNOWN_PROTOCOLS}
default_services['rpc'] = {ServicePart.HOST: 'localhost', ServicePart.PORT: 8000}
services = self._parse_services(self.default('SERVICES', ''), default_part)
# Find onion hosts
for service in services:
if str(service.host).endswith('.onion'):
raise ServiceError(f'bad host for SERVICES: {service}')
return services
def services_to_report(self):
services = self._parse_services(self.default('REPORT_SERVICES', ''), None)
for service in services:
if service.protocol == 'rpc':
raise ServiceError(f'bad protocol for REPORT_SERVICES: {service.protocol}')
if isinstance(service.host, (IPv4Address, IPv6Address)):
ip_addr = service.host
if (ip_addr.is_multicast or ip_addr.is_unspecified or
(ip_addr.is_private and self.peer_announce)):
raise ServiceError(f'bad IP address for REPORT_SERVICES: {ip_addr}')
elif service.host.lower() == 'localhost':
raise ServiceError(f'bad host for REPORT_SERVICES: {service.host}')
return services
def peer_discovery_enum(self):
pd = self.default('PEER_DISCOVERY', 'on').strip().lower()
if pd in ('off', ''):
return self.PD_OFF
elif pd == 'self':
return self.PD_SELF
else:
return self.PD_ON
def db_engine_enum(self) -> str:
from electrumx.server.storage import list_db_engine_choices
choices = list_db_engine_choices()
db_eng = self.default('DB_ENGINE', None)
if db_eng is None:
raise self.Error(
f"required envvar DB_ENGINE not set.\n"
f"Choose one from: {choices}.\n"
f"The corresponding dependencies also need to be installed. See pip extras.\n"
f"In ElectrumX 1.x versions, the default was leveldb.\n"
f"Warning: It is not possible to switch back and forth, "
f"the on-disk DB formats are not compatible with each other: when switching, "
f"you have to resync from genesis.\n"
f"LevelDB was written with HDDs in mind. RocksDB is more modern and performs better (on an SSD)."
)
db_eng = db_eng.lower()
if db_eng in choices:
return db_eng
else:
raise self.Error(
f"bad value for envvar DB_ENGINE: {db_eng!r}. Choose one from: {choices}."
)
src/electrumx/server/history.py
+210
View File
@@ -0,0 +1,210 @@
# Copyright (c) 2016-2018, Neil Booth
# Copyright (c) 2017, the ElectrumX authors
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''History by script hash (address).'''
import ast
import bisect
import time
from array import array
from collections import defaultdict
from typing import TYPE_CHECKING, Type, Optional, Sequence, Iterable
import electrumx.lib.util as util
from electrumx.lib.hash import HASHX_LEN
from electrumx.server.db_util import (
DBTooOldForMigrations,
pack_txnum, unpack_txnum, TXNUM_LEN,
)
if TYPE_CHECKING:
from electrumx.server.storage import Storage
class History:
DB_VERSIONS = (4, )
DB_STATE_KEY = b'state\0\0'
db: Optional['Storage']
def __init__(self):
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.unflushed = set() # type: set[bytes]
self.hist_db_tx_count = 0
self.hist_db_tx_count_next = 0 # after next flush, next value for self.hist_db_tx_count
self.db_version = max(self.DB_VERSIONS)
self.upgrade_cursor = -1
# Key: b'H' + address_hashX + tx_num
# Value: <null>
self.db = None
def open_db(
self,
*,
db_class: Type['Storage'],
for_sync: bool,
utxo_db_tx_count: int,
) -> None:
self.db = db_class('hist', for_sync)
self.read_state()
self.clear_excess(utxo_db_tx_count)
def close_db(self) -> None:
if self.db:
self.db.close()
self.db = None
def read_state(self) -> None:
state = self.db.get(self.DB_STATE_KEY)
if state:
state = ast.literal_eval(state.decode())
if not isinstance(state, dict):
raise RuntimeError('failed reading state from history DB')
self.db_version = state.get('db_version', 0)
self.upgrade_cursor = state.get('upgrade_cursor', -1)
self.hist_db_tx_count = state.get('hist_db_tx_count', 0)
self.hist_db_tx_count_next = self.hist_db_tx_count
if self.db_version not in self.DB_VERSIONS:
raise DBTooOldForMigrations(
db_name="history", db_version=self.db_version, supported_versions=self.DB_VERSIONS)
if self.db_version != max(self.DB_VERSIONS):
raise Exception("missing db upgrade") # call future upgrade logic here
self.logger.info(f'history DB version: {self.db_version}')
def clear_excess(self, utxo_db_tx_count: int) -> None:
# self.hist_db_tx_count != utxo_db_tx_count might happen as
# both DBs cannot be updated atomically.
# However, we MUST maintain the invariant that hist_db_tx_count >= utxo_db_tx_count:
# - when advancing blocks, hist_db is flushed first, so its count can be higher;
# but when backing up (during reorg), hist_db is flushed last, so its count can be higher.
if self.hist_db_tx_count == utxo_db_tx_count:
return # happy path
elif self.hist_db_tx_count < utxo_db_tx_count:
raise RuntimeError(f"corrupted DB. {self.hist_db_tx_count=} < {utxo_db_tx_count=}")
assert self.hist_db_tx_count > utxo_db_tx_count
self.logger.info(
'DB shut down uncleanly. Scanning for excess history flushes... '
'This might take a loooong time. :/'
)
utxo_db_tx_count_bytes = pack_txnum(utxo_db_tx_count)
keys = [] # FIXME could grow large in-memory?
for iter_cnt, (db_key, _db_val) in enumerate(self.db.iterator(prefix=b'H')):
if iter_cnt % 10_000_000 == 0:
self.logger.info(
f'Scanning for excess history flushes... num history entries checked: '
f'{iter_cnt // 1_000_000} million. will delete: {len(keys):,d}')
tx_numb = db_key[-TXNUM_LEN:]
# we exploit that pack_txnum uses big-endian byte-order, and just compare the bytes directly:
# tx_num = unpack_txnum(tx_numb)
if tx_numb >= utxo_db_tx_count_bytes:
keys.append(db_key)
self.logger.info(f'deleting {len(keys):,d} history entries')
self.hist_db_tx_count = utxo_db_tx_count
self.hist_db_tx_count_next = self.hist_db_tx_count
with self.db.write_batch() as batch:
for key in keys:
batch.delete(key)
self._write_state(batch)
self.logger.info('deleted excess history entries')
def _write_state(self, batch) -> None:
'''Write state to the history DB.'''
state = {
'hist_db_tx_count': self.hist_db_tx_count,
'db_version': self.db_version,
'upgrade_cursor': self.upgrade_cursor,
}
batch.put(self.DB_STATE_KEY, repr(state).encode())
def add_unflushed(
self,
hashXs_by_tx: list[list[bytes]],
first_tx_num: int,
) -> None:
"""This method must be thread-safe. (runs on bp.pool_executor)"""
unflushed = self.unflushed
for tx_num, hashXs in enumerate(hashXs_by_tx, start=first_tx_num):
tx_numb = pack_txnum(tx_num)
for hashX in hashXs:
unflushed.add(hashX + tx_numb)
def update_tx_count_next(self, tx_count: int) -> None:
self.hist_db_tx_count_next = tx_count
def unflushed_memsize(self) -> int:
# note: the magic numbers here were estimated using util.deep_getsizeof
return len(self.unflushed) * 85 + len(self.unflushed) * (HASHX_LEN + TXNUM_LEN)
def assert_flushed(self) -> None:
assert not self.unflushed
def flush(self) -> None:
start_time = time.monotonic()
unflushed = self.unflushed
with self.db.write_batch() as batch:
for hashX_txnum in sorted(unflushed):
db_key = b'H' + hashX_txnum
batch.put(db_key, b'')
self.hist_db_tx_count = self.hist_db_tx_count_next
self._write_state(batch)
count = len(unflushed)
unflushed.clear()
if self.db.for_sync:
elapsed = time.monotonic() - start_time
self.logger.info(f'flushed history in {elapsed:.1f}s '
f'for {count:,d} history entries')
def backup(self, *, hashXs: Iterable[bytes], tx_count: int) -> None:
self.assert_flushed()
nremoves = 0
with self.db.write_batch() as batch:
for hashX in sorted(hashXs):
deletes = []
prefix = b'H' + hashX
for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True):
tx_numb = db_key[-TXNUM_LEN:]
tx_num = unpack_txnum(tx_numb)
if tx_num >= tx_count:
nremoves += 1
deletes.append(db_key)
else:
# note: we can break now, due to 'reverse=True' and txnums being big endian
break
for key in deletes:
batch.delete(key)
self.hist_db_tx_count = tx_count
self.hist_db_tx_count_next = self.hist_db_tx_count
self._write_state(batch)
self.logger.info(f'backing up removed {nremoves:,d} history entries')
def get_txnums(self, hashX: bytes, limit: int = 1000) -> Iterable[int]:
'''Generator that returns an unpruned, sorted list of tx_nums in the
history of a hashX. Includes both spending and receiving
transactions. By default yields at most 1000 entries. Set
limit to None to get them all. '''
limit = util.resolve_limit(limit)
prefix = b'H' + hashX
for db_key, db_val in self.db.iterator(prefix=prefix):
tx_numb = db_key[-TXNUM_LEN:]
if limit == 0:
return
tx_num = unpack_txnum(tx_numb)
yield tx_num
limit -= 1
src/electrumx/server/mempool.py
+626
View File
@@ -0,0 +1,626 @@
# Copyright (c) 2016-2018, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Mempool handling.'''
import itertools
import time
from abc import ABC, abstractmethod
import asyncio
from collections import defaultdict
from dataclasses import dataclass
from typing import Sequence, Tuple, TYPE_CHECKING, Type, Dict, Optional, Set, Iterable
import math
from graphlib import TopologicalSorter
from aiorpcx import run_in_thread, sleep, ignore_after
from electrumx.lib.hash import hash_to_hex_str, hex_str_to_hash
from electrumx.lib.tx import SkipTxDeserialize
from electrumx.lib.util import class_logger, chunks, OldTaskGroup, LogTimeTaken
from electrumx.lib.tx import TXOSpendStatus, TxOutpoint
from electrumx.server.db import UTXO
if TYPE_CHECKING:
from electrumx.lib.coins import Coin
DB_UTXO_MAP = dict[TxOutpoint, Optional[tuple[bytes, int]]] # prevout->(hashX,value_in_sats)
@dataclass(slots=True)
class MemPoolTx:
prevouts: Sequence[TxOutpoint] # (txid_rev, txout_idx)
# A pair is a (hashX, value) tuple
in_pairs: Optional[Sequence[tuple[bytes, int]]] # (hashX, value_in_sats)
out_pairs: Sequence[tuple[bytes, int]] # (hashX, value_in_sats)
fee: int # in sats
size: int # in vbytes
@dataclass(slots=True)
class MemPoolTxSummary:
txid_rev: bytes
fee: int # in sats
has_unconfirmed_inputs: bool
@dataclass(slots=True, frozen=True, kw_only=True)
class RecentMemPoolTx:
txid_rev: bytes
fee: int # in sats
vsize: int # in vbytes
class DBSyncError(Exception):
pass
class MemPoolAPI(ABC):
'''A concrete instance of this class is passed to the MemPool object
and used by it to query DB and blockchain state.'''
@abstractmethod
async def height(self) -> int:
'''Query bitcoind for its height.'''
@abstractmethod
def cached_height(self) -> Optional[int]:
'''Return the height of bitcoind the last time it was queried,
for any reason, without actually querying it.
'''
@abstractmethod
async def daemon_height_changed(self) -> None:
'''Wait until bitcoind's height changes.'''
@abstractmethod
def db_height(self) -> int:
'''Return the height flushed to the on-disk DB.'''
@abstractmethod
async def db_height_changed(self) -> None:
'''Wait until the on-disk DB height changes.'''
@abstractmethod
async def mempool_txids_hum(self) -> Sequence[str]:
'''Query bitcoind for the txids of all transactions in its
mempool, returned as a list.'''
@abstractmethod
async def raw_transactions(self, txids_hum: Iterable[str]) -> Sequence[bytes | None]:
'''Query bitcoind for the serialized raw transactions with the given
txids. Missing transactions are returned as None.
txids_hum is an iterable of hexadecimal hash strings.'''
@abstractmethod
async def lookup_utxos(self, prevouts: Sequence[TxOutpoint]) -> Sequence[Optional[Tuple[bytes, int]]]:
'''Return a list of (hashX, value) pairs, one for each prevout if unspent,
otherwise return None if spent or not found (for the given prevout).
prevouts - an iterable of (txid_rev, txout_idx) pairs
'''
@abstractmethod
async def on_mempool(
self,
*,
touched_hashxs: Set[bytes],
touched_outpoints: Set[TxOutpoint],
height: int,
):
'''Called each time the mempool is synchronized. touched_hashxs and
touched_outpoints are sets of hashXs and tx outpoints touched since
the previous call. height is the daemon's height at the time the
mempool was obtained.
'''
class MemPool:
'''Representation of the daemon's mempool.
coin - a coin class from coins.py
api - an object implementing MemPoolAPI
Updated regularly in caught-up state. Goal is to enable efficient
response to the calls in the external interface. To that end we
maintain the following maps:
tx: txid_rev -> MemPoolTx
hashXs: hashX -> set of all txids_rev of txs touching the hashX
'''
def __init__(
self,
coin: Type['Coin'],
api: MemPoolAPI,
*,
refresh_secs=5.0,
log_status_secs=60.0,
):
assert isinstance(api, MemPoolAPI)
self.coin = coin
self.api = api
self.logger = class_logger(__name__, self.__class__.__name__)
self.txs = {} # type: Dict[bytes, MemPoolTx] # txid_rev->tx
self.hashXs = defaultdict(set) # type: Dict[Optional[bytes], Set[bytes]] # hashX->txids_rev
self.txo_to_spender = {} # type: Dict[TxOutpoint, bytes] # prevout->txid_rev
self.cached_compact_histogram = [] # type: Sequence[tuple[float, int]]
self.refresh_secs = refresh_secs
self.log_status_secs = log_status_secs
# Prevents mempool refreshes during fee histogram calculation
self.lock = asyncio.Lock()
async def _logging(self, synchronized_event):
'''Print regular logs of mempool stats.'''
self.logger.info('beginning processing of daemon mempool. '
'This can take some time...')
start = time.monotonic()
await synchronized_event.wait()
elapsed = time.monotonic() - start
self.logger.info(f'synced in {elapsed:.2f}s')
while True:
mempool_size = sum(tx.size for tx in self.txs.values()) / 1_000_000
self.logger.info(f'{len(self.txs):,d} txs {mempool_size:.2f} MB, '
f'touching {len(self.hashXs):,d} addresses. '
f'{len(self.txo_to_spender):,d} spends.')
await sleep(self.log_status_secs)
await synchronized_event.wait()
async def _refresh_histogram(self, synchronized_event):
while True:
await synchronized_event.wait()
async with self.lock:
# Threaded as can be expensive
bin_size = self.coin.MEMPOOL_COMPACT_HISTOGRAM_BINSIZE
await run_in_thread(self._update_histogram, bin_size)
await sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
def _update_histogram(self, bin_size):
# Build a histogram by fee rate
t0 = time.monotonic()
histogram = defaultdict(int)
for tx in self.txs.values():
fee_rate = tx.fee / tx.size
# use 0.1 sat/byte resolution
# note: rounding *down* is intentional. This ensures txs
# with a given fee rate will end up counted in the expected
# bucket/interval of the compact histogram.
fee_rate = math.floor(10 * fee_rate) / 10
histogram[fee_rate] += tx.size
compact = self._compress_histogram(histogram, bin_size=bin_size)
self.logger.info(f'compact fee histogram: {compact}. took {time.monotonic() - t0:.3f}s.')
self.cached_compact_histogram = compact
@classmethod
def _compress_histogram(
cls, histogram: Dict[float, int], *, bin_size: int
) -> Sequence[Tuple[float, int]]:
'''Calculate and return a compact fee histogram as needed for
"mempool.get_fee_histogram" protocol request.
histogram: feerate (sat/vbyte) -> total size in bytes of txs that pay approx feerate
bin_size: ~minimum vsize of a bucket of txs in the result (e.g. 100 kb)
'''
# Now compact it. For efficiency, get_fees returns a
# compact histogram with variable bin size. The compact
# histogram is an array of (fee_rate, vsize) values.
# vsize_n is the cumulative virtual size of mempool
# transactions with a fee rate in the interval
# [rate_(n-1), rate_n)], and rate_(n-1) > rate_n.
# Intervals are chosen to create tranches containing at
# least a certain cumulative size (bin_size) of transactions.
assert bin_size > 0
compact = []
cum_size = 0
prev_fee_rate = None
for fee_rate, size in sorted(histogram.items(), reverse=True):
# if there is a big lump of txns at this specific size,
# consider adding the previous item now (if not added already)
if size > 2 * bin_size and prev_fee_rate is not None and cum_size > 0:
compact.append((prev_fee_rate, cum_size))
cum_size = 0
bin_size *= 1.1
# now consider adding this item
cum_size += size
if cum_size > bin_size:
compact.append((fee_rate, cum_size))
cum_size = 0
bin_size *= 1.1
prev_fee_rate = fee_rate
return compact
def _accept_transactions(
self,
*,
tx_map: Dict[bytes, MemPoolTx], # txid_rev->tx
utxo_map: DB_UTXO_MAP, # prevout->(hashX,value_in_sats)
touched_hashxs: Set[bytes], # set of hashXs
touched_outpoints: Set[TxOutpoint], # set of outpoints
topologically_sort: bool,
) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]:
'''Accept transactions in tx_map to the mempool if all their inputs
can be found in the existing mempool or a utxo_map from the
DB.
Returns an (unprocessed tx_map, unspent utxo_map) pair.
'''
hashXs = self.hashXs
txs = self.txs
txo_to_spender = self.txo_to_spender
if topologically_sort:
# Sort tx_map so that parent txs come first (in case of unconf chain).
# This is just an optimization so that fewer txs will get "deferred", leading to better perf.
# Dependencies outside tx_map (already in chainstate (utxo_map), or our mempool (self.txs),
# or MISSING) are ignored.
cand_to_parents = {
txid: {parent_txid for (parent_txid, txout_idx) in tx.prevouts
if parent_txid in tx_map}
for (txid, tx) in tx_map.items()}
cand_txids = list(TopologicalSorter(cand_to_parents).static_order())
assert len(tx_map) == len(cand_to_parents) == len(cand_txids), \
f"{len(tx_map)=}, {len(cand_to_parents)=}, {len(cand_txids)=}"
del cand_to_parents
else:
cand_txids = tx_map.keys()
deferred = {} # type: dict[bytes, MemPoolTx]
unspent = set(utxo_map)
# Try to find all prevouts so we can accept the candidate TXs from tx_map into our mempool
for txid_rev in cand_txids:
tx = tx_map[txid_rev]
in_pairs = []
try:
for prevout in tx.prevouts:
# first, look for parent tx among confirmed UTXOs:
utxo = utxo_map.get(prevout)
if not utxo: # second, look for parent tx in mempool
prev_hash, prev_index = prevout
# Raises KeyError if prev_hash is not in txs
utxo = txs[prev_hash].out_pairs[prev_index]
in_pairs.append(utxo)
except KeyError:
deferred[txid_rev] = tx
continue
# Spend the prevouts
unspent.difference_update(tx.prevouts)
# Save the in_pairs, compute the fee and accept the TX
tx.in_pairs = tuple(in_pairs)
# Avoid negative fees if dealing with generation-like transactions
# because some in_parts would be missing
tx.fee = max(0, (sum(v for _, v in tx.in_pairs) -
sum(v for _, v in tx.out_pairs)))
txs[txid_rev] = tx
for hashX, _value in itertools.chain(tx.in_pairs, tx.out_pairs):
touched_hashxs.add(hashX)
hashXs[hashX].add(txid_rev)
for prevout in tx.prevouts:
txo_to_spender[prevout] = txid_rev
touched_outpoints.add(prevout)
for out_idx, out_pair in enumerate(tx.out_pairs):
touched_outpoints.add((txid_rev, out_idx))
return deferred, {prevout: utxo_map[prevout] for prevout in unspent}
async def _refresh_hashes(self, synchronized_event):
'''Refresh our view of the daemon's mempool.'''
# touched_* accumulates between calls to on_mempool and each
# call transfers ownership
touched_hashxs = set()
touched_outpoints = set()
prev_mempool_height = -1
while True:
height = self.api.cached_height()
is_new_block = height != prev_mempool_height
with LogTimeTaken(self.logger, "getrawmempool RPC", enabled=is_new_block):
txids_hum = await self.api.mempool_txids_hum()
if height != await self.api.height(): # if height changed *again*, re-start
continue
with LogTimeTaken(self.logger, "convert_txids", enabled=is_new_block):
txids_rev = await run_in_thread(lambda: {hex_str_to_hash(hh) for hh in txids_hum})
if self.api.db_height() < height:
# DB not yet caught up. give it some time...
async with ignore_after(self.refresh_secs):
await self.api.db_height_changed()
try:
async with self.lock:
await self._process_mempool(
all_txids_rev=txids_rev,
touched_hashxs=touched_hashxs,
touched_outpoints=touched_outpoints,
mempool_height=height,
debug_logs=is_new_block,
)
except DBSyncError:
# The UTXO DB is not at the same height as the
# mempool; wait and try again
self.logger.debug('waiting for DB to sync')
else:
synchronized_event.set()
synchronized_event.clear()
await self.api.on_mempool(
touched_hashxs=touched_hashxs,
touched_outpoints=touched_outpoints,
height=height,
)
prev_mempool_height = height
touched_hashxs = set()
touched_outpoints = set()
# poll bitcoind's whole mempool every few seconds; or instantly after the DB height changes
async with ignore_after(self.refresh_secs):
async with OldTaskGroup(wait=any) as group:
await group.spawn(self.api.db_height_changed())
await group.spawn(self.api.daemon_height_changed())
async def _process_mempool(
self,
*,
all_txids_rev: Set[bytes], # set of txids_rev # complete view of daemon's mempool
touched_hashxs: Set[bytes], # set of hashXs
touched_outpoints: Set[TxOutpoint], # set of outpoints
mempool_height: int,
debug_logs: bool = False,
) -> None:
# Re-sync with the new set of hashes
txs = self.txs
hashXs = self.hashXs
txo_to_spender = self.txo_to_spender
if mempool_height != self.api.db_height(): # FIXME should compare blockhash
raise DBSyncError
# 1. Handle txs that have disappeared (evicted, just got mined, etc)
# TODO split disappeared txs workload into a threadpool, chunks of ~200 txs
def handle_disappeared_txs() -> None:
nonlocal touched_hashxs
for txid_rev in disappeared_hashes:
tx = txs.pop(txid_rev)
# hashXs
tx_hashXs = {hashX for hashX, value in tx.in_pairs}
tx_hashXs.update(hashX for hashX, value in tx.out_pairs)
for hashX in tx_hashXs:
hashXs[hashX].remove(txid_rev)
if not hashXs[hashX]:
del hashXs[hashX]
touched_hashxs |= tx_hashXs
# outpoints
for prevout in tx.prevouts:
del txo_to_spender[prevout]
touched_outpoints.add(prevout)
for out_idx, out_pair in enumerate(tx.out_pairs):
touched_outpoints.add((txid_rev, out_idx))
with LogTimeTaken(self.logger, "_process_mempool() removing txs", enabled=debug_logs):
disappeared_hashes = await run_in_thread(lambda: set(txs) - all_txids_rev)
handle_disappeared_txs()
# 2. Process new transactions
t0 = time.monotonic()
new_hashes = await run_in_thread(lambda: list(all_txids_rev.difference(txs)))
if new_hashes:
# 2.1. fetch raw txs from bitcoin daemon
group = OldTaskGroup()
for hashes in chunks(new_hashes, 200):
coro = self._fetch_raw_txs_and_utxos(
new_txids_rev=hashes,
all_txids_rev=all_txids_rev,
)
await group.spawn(coro)
if mempool_height != self.api.db_height():
raise DBSyncError
tx_map = {} # type: dict[bytes, MemPoolTx]
utxo_map = {} # type: DB_UTXO_MAP
async for task in group:
partial_tx_map, partial_utxo_map = task.result()
tx_map.update(partial_tx_map)
utxo_map.update(partial_utxo_map)
if debug_logs:
self.logger.debug(f'_process_mempool() fetching txs took {time.monotonic() - t0:.3f}s. {len(new_hashes)=}')
# 2.2. accept txs into our mempool
def accept_txs_loop() -> None:
# Accept candidate txs from tx_map into our mempool.
# We only accept candidates for which we can find a UTXO for each tx input:
# - some UTXOs we find in the DB=utxo_map (chainstate),
# - some we find in self.txs (our memool),
# - some we might only find in tx_map (among the candidates: consider long chain of unconfirmed txs)
# - some we might not find anywhere: if DB is corrupted or bitcoind gave inconsistent mempool, or other race
# In each iteration, we loop over tx_map and move accepted transactions from it to self.txs.
# In the worst degenerate case, this could be O(n^2). To avoid that, we topologically sort tx_map.
nonlocal tx_map, utxo_map
prior_txmap_size = 0
while tx_map and len(tx_map) != prior_txmap_size:
prior_txmap_size = len(tx_map)
tx_map, utxo_map = self._accept_transactions(
tx_map=tx_map,
utxo_map=utxo_map,
touched_hashxs=touched_hashxs,
touched_outpoints=touched_outpoints,
topologically_sort=True,
)
if tx_map:
self.logger.error(f'{len(tx_map)} txs dropped')
with LogTimeTaken(self.logger, f"_process_mempool() accepting txs ({len(new_hashes)=})", enabled=debug_logs):
accept_txs_loop()
async def _fetch_raw_txs_and_utxos(
self,
*,
new_txids_rev: set[bytes], # (some) new candidate txs for our mempool
all_txids_rev: set[bytes], # complete view of daemon's mempool
) -> tuple[dict[bytes, MemPoolTx], DB_UTXO_MAP]:
'''Fetch a list of mempool transactions, and lookup corresponding UTXOs in the DB.'''
txids_hum_iter = (hash_to_hex_str(hash) for hash in new_txids_rev)
raw_txs = await self.api.raw_transactions(txids_hum_iter)
def deserialize_txs() -> Dict[bytes, MemPoolTx]:
"""This function is pure"""
to_hashX = self.coin.hashX_from_script
deserializer = self.coin.DESERIALIZER
txs = {} # type: Dict[bytes, MemPoolTx]
for txid_rev, raw_tx in zip(new_txids_rev, raw_txs):
# The daemon may have evicted the tx from its
# mempool or it may have gotten in a block
if not raw_tx:
continue
try:
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
except SkipTxDeserialize as ex:
self.logger.debug(f'skipping tx {hash_to_hex_str(txid_rev)}: {ex}')
continue
# Convert the inputs and outputs into (hashX, value) pairs
# Drop generation-like inputs from MemPoolTx.prevouts
txin_pairs = tuple((txin.prev_txid_rev, txin.prev_idx)
for txin in tx.inputs
if not txin.is_generation())
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
for txout in tx.outputs)
txs[txid_rev] = MemPoolTx(
prevouts=txin_pairs,
in_pairs=None,
out_pairs=txout_pairs,
fee=0,
size=tx_size,
)
return txs
# Thread this potentially slow operation so as not to block
tx_map = await run_in_thread(deserialize_txs) # type: Dict[bytes, MemPoolTx]
# Determine all prevouts not in the mempool, and fetch the
# UTXO information from the database. Failed prevout lookups
# return None - concurrent database updates happen - which is
# relied upon by _accept_transactions. Ignore prevouts that are
# generation-like.
prevouts = tuple(prevout for tx in tx_map.values()
for prevout in tx.prevouts
if prevout[0] not in all_txids_rev)
utxos = await self.api.lookup_utxos(prevouts)
utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)}
return tx_map, utxo_map
#
# External interface
#
async def keep_synchronized(self, synchronized_event: asyncio.Event) -> None:
'''Keep the mempool synchronized with the daemon.'''
async with OldTaskGroup() as group:
await group.spawn(self._refresh_hashes(synchronized_event))
await group.spawn(self._refresh_histogram(synchronized_event))
await group.spawn(self._logging(synchronized_event))
async def balance_delta(self, hashX: bytes) -> int:
'''Return the unconfirmed amount in the mempool for hashX.
Can be positive or negative.
'''
value = 0
if hashX in self.hashXs:
for txid_rev in self.hashXs[hashX]:
tx = self.txs[txid_rev]
value -= sum(v for h168, v in tx.in_pairs if h168 == hashX)
value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
return value
async def compact_fee_histogram(self) -> Sequence[tuple[float, int]]:
'''Return a compact fee histogram of the current mempool.'''
return self.cached_compact_histogram
async def potential_spends(self, hashX: bytes) -> set[TxOutpoint]:
'''Return a set of (prev_hash, prev_idx) pairs from mempool
transactions that touch hashX.
None, some or all of these may be spends of the hashX, but all
actual spends of it (in the DB or mempool) will be included.
'''
result = set()
for txid_rev in self.hashXs.get(hashX, ()):
tx = self.txs[txid_rev]
result.update(tx.prevouts)
return result
async def transaction_summaries(self, hashX: bytes) -> Sequence[MemPoolTxSummary]:
'''Return a list of MemPoolTxSummary objects for the hashX,
sorted as expected by protocol methods.
'''
result = [] # type: list[MemPoolTxSummary]
for txid_rev in self.hashXs.get(hashX, ()):
tx = self.txs[txid_rev]
has_ui = any(hash in self.txs for hash, idx in tx.prevouts)
result.append(MemPoolTxSummary(txid_rev, tx.fee, has_ui))
result.sort(key=lambda x: (x.has_unconfirmed_inputs, x.txid_rev[::-1]))
return result
async def unordered_UTXOs(self, hashX: bytes) -> Sequence[UTXO]:
'''Return an unordered list of UTXO named tuples from mempool
transactions that pay to hashX.
This does not consider if any other mempool transactions spend
the outputs.
'''
utxos = []
for txid_rev in self.hashXs.get(hashX, ()):
tx = self.txs.get(txid_rev)
for pos, (hX, value) in enumerate(tx.out_pairs):
if hX == hashX:
utxos.append(UTXO(-1, pos, txid_rev, 0, value))
return utxos
async def spender_for_txo(self, prev_txid_rev: bytes, txout_idx: int) -> 'TXOSpendStatus':
'''For an outpoint, returns its spend-status.
This only considers the mempool, not the DB/blockchain, so e.g. mined
txs are not distinguished from txs that never existed.
'''
# look up funding tx
prev_tx = self.txs.get(prev_txid_rev, None)
if prev_tx is None:
# funding tx already mined or never existed
funder_height = None
else:
if len(prev_tx.out_pairs) <= txout_idx:
# output idx out of bounds...?
return TXOSpendStatus(funder_height=None)
prev_has_ui = any(hash in self.txs for hash, idx in prev_tx.prevouts)
funder_height = -prev_has_ui
prevout = (prev_txid_rev, txout_idx)
# look up spending tx
spender_txid_rev = self.txo_to_spender.get(prevout, None)
if spender_txid_rev is None:
return TXOSpendStatus(funder_height=funder_height)
spender_tx = self.txs.get(spender_txid_rev, None)
if spender_tx is None:
self.logger.warning(f"spender_tx {hash_to_hex_str(spender_txid_rev)} not in"
f"mempool, but txo_to_spender referenced it as spender "
f"of {hash_to_hex_str(prev_txid_rev)}:{txout_idx} ?!")
return TXOSpendStatus(funder_height=funder_height)
spender_has_ui = any(txid_rev in self.txs for txid_rev, idx in spender_tx.prevouts)
spender_height = -spender_has_ui
return TXOSpendStatus(
funder_height=funder_height,
spender_txid_rev=spender_txid_rev,
spender_height=spender_height,
)
async def get_recently_added_txs(self, *, count: int) -> Sequence[RecentMemPoolTx]:
# note: inefficient for large "count"s
it = reversed(self.txs.items())
count = min(count, len(self.txs))
mempool_txs = [next(it) for _ in range(count)]
return [
RecentMemPoolTx(txid_rev=hash, fee=mtx.fee, vsize=mtx.size)
for hash, mtx in mempool_txs]
src/electrumx/server/peers.py
+634
View File
@@ -0,0 +1,634 @@
# Copyright (c) 2017-2018, Neil Booth
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Peer management.'''
import asyncio
import random
import socket
import ssl
import time
from collections import Counter, defaultdict
from ipaddress import IPv4Address, IPv6Address
from typing import TYPE_CHECKING, Type, Any, Optional
from functools import partial
import aiohttp
from aiorpcx import (Event, Notification, RPCSession, SOCKSError,
SOCKSProxy, TaskTimeout, connect_rs,
handler_invocation, ignore_after, sleep)
from aiorpcx.jsonrpc import CodeMessageError
from electrumx.lib.peer import Peer
from electrumx.lib.util import class_logger, json_deserialize, OldTaskGroup
from electrumx.server.session import RPCSessionWithTaskGroup
from electrumx.server.transport import PaddedRSTransport
if TYPE_CHECKING:
from electrumx.server.env import Env
from electrumx.server.db import DB
PEER_GOOD, PEER_STALE, PEER_NEVER, PEER_BAD = range(4)
STATUS_DESCS = ('good', 'stale', 'never', 'bad')
STALE_SECS = 3 * 3600
WAKEUP_SECS = 300
PEER_ADD_PAUSE = 600
class BadPeerError(Exception):
pass
def assert_good(message, result, instance):
if not isinstance(result, instance):
raise BadPeerError(f'{message} returned bad result type '
f'{type(result).__name__}')
class PeerSession(RPCSessionWithTaskGroup):
'''An outgoing session to a peer.'''
async def handle_request(self, request):
# We subscribe so might be unlucky enough to get a notification...
if (isinstance(request, Notification) and
request.method == 'blockchain.headers.subscribe'):
pass
else:
await handler_invocation(None, request) # Raises
class PeerManager:
'''Looks after the DB of peer network servers.
Attempts to maintain a connection with up to 8 peers.
Issues a 'peers.subscribe' RPC to them and tells them our data.
'''
def __init__(self, env: 'Env', db: 'DB'):
self.logger = class_logger(__name__, self.__class__.__name__)
# Initialise the Peer class
Peer.DEFAULT_PORTS = env.coin.PEER_DEFAULT_PORTS
self.env = env
self.db = db
# Our reported clearnet and Tor Peers, if any
sclass = env.coin.SESSIONCLS
self.myselves = [Peer(str(service.host), sclass.server_features(env), 'env')
for service in env.report_services]
self.server_version_args = sclass.server_version_args()
# Peers have one entry per hostname. Once connected, the
# ip_addr property is either None, an onion peer, or the
# IP address that was connected to. Adding a peer will evict
# any other peers with the same host name or IP address.
self.peers = set()
self.permit_onion_peer_time = time.time()
self.proxy = None
self.group = OldTaskGroup()
self.recent_peer_adds = {}
# refreshed
self.blacklist = set()
def _my_clearnet_peer(self):
'''Returns the clearnet peer representing this server, if any.'''
clearnet = [peer for peer in self.myselves if not peer.is_tor]
return clearnet[0] if clearnet else None
def _set_peer_statuses(self):
'''Set peer statuses.'''
cutoff = time.time() - STALE_SECS
for peer in self.peers:
if peer.bad:
peer.status = PEER_BAD
elif peer.last_good > cutoff:
peer.status = PEER_GOOD
elif peer.last_good:
peer.status = PEER_STALE
else:
peer.status = PEER_NEVER
def _features_to_register(self, peer, remote_peers):
'''If we should register ourselves to the remote peer, which has
reported the given list of known peers, return the clearnet
identity features to register, otherwise None.
'''
# Announce ourself if not present. Don't if disabled, we
# are a non-public IP address, or to ourselves.
if not self.env.peer_announce or peer in self.myselves:
return None
my = self._my_clearnet_peer()
if not my or not my.is_public:
return None
# Register if no matches, or ports have changed
for peer in my.matches(remote_peers):
if peer.tcp_port == my.tcp_port and peer.ssl_port == my.ssl_port:
return None
return my.features
def _permit_new_onion_peer(self, now):
'''Accept a new onion peer only once per random time interval.'''
if now < self.permit_onion_peer_time:
return False
self.permit_onion_peer_time = now + random.randrange(0, 1200)
return True
async def _import_peers(self):
'''Import hard-coded peers from a file or the coin defaults.'''
imported_peers = self.myselves.copy()
# Add the hard-coded ones unless only reporting ourself
if self.env.peer_discovery != self.env.PD_SELF:
imported_peers.extend(Peer.from_real_name(real_name, 'coins.py')
for real_name in self.env.coin.PEERS)
await self._note_peers(imported_peers, limit=None)
async def _refresh_blacklist(self):
url = self.env.blacklist_url
if not url:
return
async def read_blacklist():
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
text = await response.text()
return {entry.lower() for entry in json_deserialize(text)}
while True:
try:
self.blacklist = await read_blacklist()
except Exception as e:
self.logger.error(f'could not retrieve blacklist from {url}: {e}')
else:
self.logger.info(f'blacklist from {url} has {len(self.blacklist)} entries')
# Got new blacklist. Now check our current peers against it
for peer in self.peers:
if self._is_blacklisted(peer):
peer.retry_event.set()
await sleep(600)
def _is_blacklisted(self, peer):
host = peer.host.lower()
second_level_domain = '*.' + '.'.join(host.split('.')[-2:])
return any(item in self.blacklist
for item in (host, second_level_domain, peer.ip_addr))
def _get_recent_good_peers(self):
cutoff = time.time() - STALE_SECS
recent = [peer for peer in self.peers
if peer.last_good > cutoff and
not peer.bad and peer.is_public]
recent = [peer for peer in recent if not self._is_blacklisted(peer)]
return recent
async def _detect_proxy(self):
'''Detect a proxy if we don't have one and some time has passed since
the last attempt.
If found self.proxy is set to a SOCKSProxy instance, otherwise None.
'''
host = self.env.tor_proxy_host
if self.env.tor_proxy_port is None:
ports = [9050, 9150, 1080]
else:
ports = [self.env.tor_proxy_port]
while True:
self.logger.info(f'trying to detect proxy on "{host}" '
f'ports {ports}')
proxy = await SOCKSProxy.auto_detect_at_host(host, ports, None)
if proxy:
self.proxy = proxy
self.logger.info(f'detected {proxy}')
return
self.logger.info('no proxy detected, will try later')
await sleep(900)
async def _note_peers(self, peers, limit=2, check_ports=False, source=None):
'''Add a limited number of peers that are not already present.'''
new_peers = []
match_set = self.peers.copy()
for peer in peers:
if not peer.is_public or (peer.is_tor and not self.proxy):
continue
matches = peer.matches(match_set)
if matches:
if check_ports:
for match in matches:
if match.check_ports(peer):
self.logger.info(f'ports changed for {peer}')
# Retry connecting to the peer. First we will try the existing
# ports and then try the new ports. Note that check_ports above
# had a side_effect to temporarily store the new ports.
# If we manage to connect, we will call 'server.features',
# and the ports for this peer will be updated to the return values.
match.retry_event.set()
else:
match_set.add(peer)
new_peers.append(peer)
if new_peers:
source = source or new_peers[0].source
if limit:
random.shuffle(new_peers)
use_peers = new_peers[:limit]
else:
use_peers = new_peers
for peer in use_peers:
self.logger.info(f'accepted new peer {peer} from {source}')
peer.retry_event = Event()
self.peers.add(peer)
await self.group.spawn(self._monitor_peer(peer))
return True
async def _monitor_peer(self, peer):
# Stop monitoring if we were dropped (a duplicate peer)
while peer in self.peers:
if await self._should_drop_peer(peer):
self.peers.discard(peer)
break
# Figure out how long to sleep before retrying. Retry a
# good connection when it is about to turn stale, otherwise
# exponentially back off retries.
if peer.try_count == 0:
pause = STALE_SECS - WAKEUP_SECS * 2
else:
pause = WAKEUP_SECS * 2 ** peer.try_count
async with ignore_after(pause):
await peer.retry_event.wait()
peer.retry_event.clear()
async def _should_drop_peer(self, peer):
peer.try_count += 1
is_good = False
for kind, port, family in peer.connection_tuples():
peer.last_try = time.time()
kwargs = {'family': family}
if kind == 'SSL':
sslc = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
sslc.check_hostname = False
sslc.verify_mode = ssl.CERT_NONE
kwargs['ssl'] = sslc
if self.env.force_proxy or peer.is_tor:
if not self.proxy:
return
kwargs['proxy'] = self.proxy
kwargs['resolve'] = not peer.is_tor
else:
# Use our listening Host/IP for outgoing non-proxy
# connections so our peers see the correct source.
local_hosts = {service.host for service in self.env.services
if isinstance(service.host, (IPv4Address, IPv6Address))
and service.protocol != 'rpc'}
if local_hosts:
kwargs['local_addr'] = (str(local_hosts.pop()), None)
session_factory = partial(
PeerSession,
manager_taskgroup=self.group,
)
peer_text = f'[{peer}:{port} {kind}]'
try:
async with connect_rs(
peer.host, port,
session_factory=session_factory,
transport=PaddedRSTransport,
**kwargs,
) as session:
session.sent_request_timeout = 120 if peer.is_tor else 30
await self._verify_peer(session, peer)
is_good = True
break
except BadPeerError as e:
self.logger.error(f'{peer_text} marking bad: ({e})')
peer.mark_bad()
break
except CodeMessageError as e:
self.logger.error(f'{peer_text} RPC error: {e.message} '
f'({e.code})')
except (OSError, SOCKSError, ConnectionError, TaskTimeout) as e:
self.logger.info(f'{peer_text} {e}')
if is_good:
# Monotonic time would be better, but last_good and last_try are
# exported to admin RPC client.
now = time.time()
elapsed = now - peer.last_try
self.logger.info(f'{peer_text} verified in {elapsed:.1f}s')
peer.try_count = 0
peer.last_good = now
peer.source = 'peer'
# At most 2 matches if we're a host name, potentially
# several if we're an IP address (several instances
# can share a NAT).
matches = peer.matches(self.peers)
for match in matches:
if match.ip_address:
if len(matches) > 1:
self.peers.remove(match)
# Force the peer's monitoring task to exit
match.retry_event.set()
elif peer.host in match.features['hosts']:
match.update_features_from_peer(peer)
# Trim this data structure
self.recent_peer_adds = {k: v for k, v in self.recent_peer_adds.items()
if v + PEER_ADD_PAUSE < now}
else:
# Forget the peer if long-term unreachable
if peer.last_good and not peer.bad:
try_limit = 10
else:
try_limit = 3
if peer.try_count >= try_limit:
desc = 'bad' if peer.bad else 'unreachable'
self.logger.info(f'forgetting {desc} peer: {peer}')
return True
return False
async def _verify_peer(self, session, peer):
# store IP address for peer
if not peer.is_tor:
address = session.remote_address()
if address and isinstance(address.host, (IPv4Address, IPv6Address)):
peer.ip_addr = str(address.host)
if self._is_blacklisted(peer):
raise BadPeerError('blacklisted')
# Bucket good recent peers; forbid many servers from similar IPs
# FIXME there's a race here, when verifying multiple peers
# that belong to the same bucket ~simultaneously
recent_peers = self._get_recent_good_peers()
if peer in recent_peers:
recent_peers.remove(peer)
onion_peers = []
buckets = defaultdict(list)
for other_peer in recent_peers:
if other_peer.is_tor:
onion_peers.append(other_peer)
else:
buckets[other_peer.bucket_for_internal_purposes()].append(other_peer)
if peer.is_tor:
# keep number of onion peers below half of all peers,
# but up to 100 is OK regardless
if len(onion_peers) > len(recent_peers) // 2 >= 100:
raise BadPeerError('too many onion peers already')
else:
bucket = peer.bucket_for_internal_purposes()
if buckets[bucket]:
raise BadPeerError(f'too many peers already in bucket {bucket}')
# server.version goes first
message = 'server.version'
try:
result = await session.send_request(message, self.server_version_args)
except asyncio.CancelledError:
raise BadPeerError('terminated before server.version response')
assert_good(message, result, list)
# Protocol version 1.1 returns a pair with the version first
if len(result) != 2 or not all(isinstance(x, str) for x in result):
raise BadPeerError(f'bad server.version result: {result}')
server_version, _protocol_version = result
peer.server_version = server_version
peer.features['server_version'] = server_version
async with OldTaskGroup() as g:
await g.spawn(self._send_headers_subscribe(session))
await g.spawn(self._send_server_features(session, peer))
peers_task = await g.spawn(self._send_peers_subscribe
(session, peer))
# Process reported peers if remote peer is good
peers = peers_task.result()
await self._note_peers(peers)
features = self._features_to_register(peer, peers)
if features:
self.logger.info(f'registering ourself with {peer}')
# We only care to wait for the response
try:
await session.send_request('server.add_peer', [features])
except asyncio.CancelledError:
raise BadPeerError('terminated before server.add_peer response')
async def _send_headers_subscribe(self, session):
message = 'blockchain.headers.subscribe'
result = await session.send_request(message)
assert_good(message, result, dict)
# Compare heights.
MAX_TIP_HEIGHT_DIFF = 5
our_height = self.db.db_height
their_height = result.get('height')
if not isinstance(their_height, int):
raise BadPeerError(f'invalid height {their_height}')
if abs(our_height - their_height) > MAX_TIP_HEIGHT_DIFF:
raise BadPeerError(f'bad height {their_height:,d} '
f'(ours: {our_height:,d})')
# Check prior header too in case of hard fork.
# Allow short chain-splits. On Bitcoin testnet4, <6 block splits and reorgs are *very* common.
MAX_CHAINSPLIT_LEN = 10
check_height = max(0, min(our_height, their_height) - MAX_CHAINSPLIT_LEN)
raw_header = await self.db.raw_header(check_height)
ours = raw_header.hex()
message = 'blockchain.block.header'
theirs = await session.send_request(message, [check_height])
assert_good(message, theirs, str)
if ours != theirs:
raise BadPeerError(f'our header {ours} and '
f'theirs {theirs} differ')
async def _send_server_features(self, session, peer):
message = 'server.features'
features = await session.send_request(message)
assert_good(message, features, dict)
features_hosts = features.get('hosts', {})
assert_good(message, features_hosts, dict)
hosts = [host.lower() for host in features_hosts]
if self.env.coin.GENESIS_HASH != features.get('genesis_hash'):
raise BadPeerError('incorrect genesis hash')
if peer.host.lower() in hosts:
peer.update_features(features)
else:
raise BadPeerError(f'not listed in own hosts list {hosts}')
async def _send_peers_subscribe(self, session, peer):
message = 'server.peers.subscribe'
raw_peers = await session.send_request(message)
assert_good(message, raw_peers, list)
# Check the peers list we got from a remote peer.
# Each is expected to be of the form:
# [ip_addr, hostname, ['v1.0', 't51001', 's51002']]
# Call add_peer if the remote doesn't appear to know about us.
try:
real_names = [' '.join([u[1]] + u[2]) for u in raw_peers]
return [Peer.from_real_name(real_name, str(peer))
for real_name in real_names]
except Exception:
raise BadPeerError('bad server.peers.subscribe response')
#
# External interface
#
async def discover_peers(self):
'''Perform peer maintenance. This includes
1) Forgetting unreachable peers.
2) Verifying connectivity of new peers.
3) Retrying old peers at regular intervals.
'''
self.logger.info(f'peer discovery: {self.env.peer_discovery}')
if self.env.peer_discovery != self.env.PD_ON:
self.logger.info('peer discovery is disabled')
return
self.logger.info(f'announce ourself: {self.env.peer_announce}')
self.logger.info(f'my clearnet self: {self._my_clearnet_peer()}')
self.logger.info(f'force use of proxy: {self.env.force_proxy}')
self.logger.info(f'beginning peer discovery...')
try:
async with self.group as group:
await group.spawn(self._refresh_blacklist())
await group.spawn(self._detect_proxy())
await group.spawn(self._import_peers())
except Exception:
self.logger.exception(
"PeerManager.group died! Please open a bug report about this, "
"and include at least this traceback and maybe a bit more of "
"the log (if it looks relevant).")
while True:
await sleep(3600)
self.logger.warning(
"Peer discovery is not working as PeerManager.group died. "
"This is not normal. You should inspect the log.")
def info(self):
'''The number of peers.'''
self._set_peer_statuses()
counter = Counter(peer.status for peer in self.peers)
return {
'bad': counter[PEER_BAD],
'good': counter[PEER_GOOD],
'never': counter[PEER_NEVER],
'stale': counter[PEER_STALE],
'total': len(self.peers),
}
async def add_localRPC_peer(self, real_name):
'''Add a peer passed by the admin over LocalRPC.'''
await self._note_peers([Peer.from_real_name(real_name, 'RPC')], check_ports=True)
async def on_add_peer(self, features: dict[str, Any] | Any, source_addr: Optional[str]):
'''Add a peer (but only if the peer resolves to the source).'''
if self.env.peer_discovery != self.env.PD_ON:
return False
if not source_addr:
self.logger.info('ignored add_peer request: no source info')
return False
source = str(source_addr.host)
peers = Peer.peers_from_features(features, source)
if not peers:
self.logger.info('ignored add_peer request: no peers given')
return False
# Just look at the first peer, require it
peer = peers[0]
host = peer.host
now = time.time()
# Rate limit peer adds by domain to one every 10 minutes
if peer.ip_address is not None:
bucket = 'ip_addr'
else:
bucket = '.'.join(host.lower().split('.')[-2:])
last = self.recent_peer_adds.get(bucket, 0)
self.recent_peer_adds[bucket] = now
if last + PEER_ADD_PAUSE >= now:
return False
if peer.is_tor:
permit = self._permit_new_onion_peer(now)
reason = 'rate limiting'
else:
getaddrinfo = asyncio.get_event_loop().getaddrinfo
try:
infos = await getaddrinfo(host, 80, type=socket.SOCK_STREAM)
except socket.gaierror:
permit = False
reason = 'address resolution failure'
else:
permit = any(source == info[-1][0] for info in infos)
reason = 'source-destination mismatch'
if permit:
self.logger.info(f'accepted add_peer request from {source} for {host}')
await self._note_peers([peer], check_ports=True)
else:
self.logger.warning(f'rejected add_peer request from {source} '
f'for {host} ({reason})')
return permit
def on_peers_subscribe(self, is_tor):
'''Returns the server peers as a list of (ip, host, details) tuples.
We return all peers we've connected to in the last day.
Additionally, if we don't have onion routing, we return a few
hard-coded onion servers.
'''
recent = self._get_recent_good_peers()
# Always report ourselves if valid (even if not public)
cutoff = time.time() - STALE_SECS
peers = {
myself
for myself in self.myselves
if myself.last_good > cutoff
}
# Bucket the clearnet peers and select up to two from each
onion_peers = []
buckets = defaultdict(list)
for peer in recent:
if peer.is_tor:
onion_peers.append(peer)
else:
buckets[peer.bucket_for_external_interface()].append(peer)
for bucket_peers in buckets.values():
random.shuffle(bucket_peers)
peers.update(bucket_peers[:2])
# Add up to 20% onion peers (but up to 10 is OK anyway)
random.shuffle(onion_peers)
max_onion = 50 if is_tor else max(10, len(peers) // 4)
peers.update(onion_peers[:max_onion])
return [peer.to_tuple() for peer in peers]
def proxy_address(self):
'''Return the NetAddress of the proxy, if there is a proxy, otherwise
None.'''
return self.proxy.address if self.proxy else None
def rpc_data(self):
'''Peer data for the peers RPC method.'''
self._set_peer_statuses()
def peer_data(peer):
data = peer.serialize()
data['status'] = STATUS_DESCS[peer.status]
return data
def peer_key(peer):
return peer.bad, -peer.last_good
return [peer_data(peer) for peer in sorted(self.peers, key=peer_key)]
src/electrumx/server/session.py
+2773
View File
File diff suppressed because it is too large. Load diff
src/electrumx/server/storage.py
+219
View File
@@ -0,0 +1,219 @@
# Copyright (c) 2016-2017, the ElectrumX authors
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
'''Backend database abstraction.'''
import os
from functools import partial
from typing import Type, Sequence, TYPE_CHECKING
import electrumx.lib.util as util
if TYPE_CHECKING:
from rocksdb import DB as rocksdb_db
from plyvel import DB as plyvel_db
def db_class(name: str) -> Type['Storage']:
'''Returns a DB engine class.'''
for db_class in util.subclasses(Storage):
if db_class.__name__.lower() == name.lower():
db_class.import_module()
return db_class
raise RuntimeError(f'unrecognised DB engine "{name}"')
def list_db_engine_choices() -> Sequence[str]:
return [db_class.__name__.lower() for db_class in util.subclasses(Storage)]
class Storage:
'''Abstract base class of the DB backend abstraction.'''
def __init__(self, name: str, for_sync: bool):
self.is_new = not os.path.exists(name)
self.for_sync = for_sync or self.is_new
self.open(name, create=self.is_new)
@classmethod
def import_module(cls) -> None:
'''Import the DB engine module.'''
raise NotImplementedError
def open(self, name: str, create: bool):
'''Open an existing database or create a new one.'''
raise NotImplementedError
def close(self) -> None:
'''Close an existing database.'''
raise NotImplementedError
def get(self, key: bytes) -> bytes:
raise NotImplementedError
def put(self, key: bytes, value: bytes) -> None:
raise NotImplementedError
def write_batch(self):
'''Return a context manager that provides `put` and `delete`.
Changes should only be committed when the context manager
closes without an exception.
'''
raise NotImplementedError
def iterator(self, prefix: bytes = b'', reverse: bool = False):
'''Return an iterator that yields (key, value) pairs from the
database sorted by key.
If `prefix` is set, only keys starting with `prefix` will be
included. If `reverse` is True the items are returned in
reverse order.
The iterator supports .seek(key), which moves it just left of `key`.
- if forward-iterating
- if `key` is present, it will be the next item
- if `key` is not present, the next item will be the smallest still greater than `key`
- if reverse-iterating
- the next item will be the largest still smaller than `key`
'''
raise NotImplementedError
class LevelDB(Storage):
'''LevelDB database engine.'''
db: 'plyvel_db'
@classmethod
def import_module(cls):
try:
import plyvel
except ImportError as e:
raise Exception(f"{e}. For the leveldb DB-backend, you should install ElectrumX with the [leveldb] pip extra.") from e
cls.module = plyvel
def open(self, name, create):
mof = 512 if self.for_sync else 128
# Use snappy compression (the default)
self.db = self.module.DB(name, create_if_missing=create,
max_open_files=mof)
self.close = self.db.close
self.get = self.db.get
self.put = self.db.put
self.write_batch = partial(self.db.write_batch, transaction=True,
sync=True)
def iterator(self, prefix=b'', reverse=False):
return LevelDBIterator(db=self.db, prefix=prefix, reverse=reverse)
class LevelDBIterator:
'''An iterator for LevelDB.'''
def __init__(self, *, db: 'plyvel_db', prefix: bytes, reverse: bool):
self.prefix = prefix
self.iterator = db.iterator(prefix=prefix, reverse=reverse)
def __iter__(self):
return self
def __next__(self):
k, v = next(self.iterator)
if not k.startswith(self.prefix):
raise StopIteration
return k, v
def seek(self, key: bytes) -> None:
self.iterator.seek(key)
class RocksDB(Storage):
'''RocksDB database engine.'''
db: 'rocksdb_db'
@classmethod
def import_module(cls):
try:
import rocksdb
except ImportError as e:
raise Exception(f"{e}. For the rocksdb DB-backend, you should install ElectrumX with the [rocksdb] pip extra.") from e
cls.module = rocksdb
def open(self, name, create):
mof = 512 if self.for_sync else 128
# Use snappy compression (the default)
options = self.module.Options(create_if_missing=create,
use_fsync=True,
target_file_size_base=33554432,
max_open_files=mof)
self.db = self.module.DB(name, options)
self.close = self.db.close
self.get = self.db.get
self.put = self.db.put
def write_batch(self):
return RocksDBWriteBatch(self.db)
def iterator(self, prefix=b'', reverse=False):
return RocksDBIterator(db=self.db, prefix=prefix, reverse=reverse)
class RocksDBWriteBatch:
'''A write batch for RocksDB.'''
def __init__(self, db: 'rocksdb_db'):
self.batch = RocksDB.module.WriteBatch()
self.db = db
def __enter__(self):
return self.batch
def __exit__(self, exc_type, exc_val, exc_tb):
if not exc_val:
self.db.write(self.batch)
class RocksDBIterator:
'''An iterator for RocksDB.'''
def __init__(self, *, db: 'rocksdb_db', prefix: bytes, reverse: bool):
self.prefix = prefix
self._is_reverse = reverse
if reverse:
self.iterator = reversed(db.iteritems())
nxt_prefix = util.increment_byte_string(prefix)
if nxt_prefix:
self.iterator.seek(nxt_prefix)
try:
next(self.iterator)
except StopIteration:
self.iterator.seek(nxt_prefix)
else:
self.iterator.seek_to_last()
else:
self.iterator = db.iteritems()
self.iterator.seek(prefix)
def __iter__(self):
return self
def __next__(self):
k, v = next(self.iterator)
if not k.startswith(self.prefix):
raise StopIteration
return k, v
def seek(self, key: bytes) -> None:
self.iterator.seek(key)
if self._is_reverse:
try:
next(self)
except StopIteration:
self.iterator.seek_to_last()
src/electrumx/server/transport.py
+122
View File
@@ -0,0 +1,122 @@
# Copyright (c) 2025, The Electrum developers
#
# All rights reserved.
#
# See the file "LICENCE" for information about the copyright
# and warranty status of this software.
import asyncio
import time
from typing import Optional, TYPE_CHECKING
from aiorpcx.rawsocket import RSTransport
if TYPE_CHECKING:
from .session import RPCSessionWithTaskGroup
class PaddedRSTransport(RSTransport):
"""A raw socket transport that provides basic countermeasures against traffic analysis
by padding the jsonrpc payload with whitespaces to have ~uniform-size TCP packets.
(it is assumed that a network observer does not see plaintext transport contents,
due to it being wrapped e.g. in TLS)
"""
MIN_PACKET_SIZE = 1024
WAIT_FOR_BUFFER_GROWTH_SECONDS = 1.0
# amount of (unpadded) bytes sent instantly before beginning with polling.
# This makes the initial handshake where a few small messages are exchanged faster.
WARMUP_BUDGET_SIZE = 1024
session: Optional['RPCSessionWithTaskGroup']
def __init__(self, *args, **kwargs):
RSTransport.__init__(self, *args, **kwargs)
self._sbuffer = bytearray() # "send buffer"
self._sbuffer_task = None # type: Optional[asyncio.Task]
self._sbuffer_has_data_evt = asyncio.Event()
self._last_send = time.monotonic()
self._force_send = False # type: bool
# note: this does not call super().write() but is a complete reimplementation
async def write(self, message):
await self._can_send.wait()
if self.is_closing():
return
framed_message = self._framer.frame(message)
self._sbuffer += framed_message
self._sbuffer_has_data_evt.set()
self._maybe_consume_sbuffer()
def _maybe_consume_sbuffer(self) -> None:
"""Maybe take some data from sbuffer and send it on the wire."""
if not self._can_send.is_set() or self.is_closing():
return
buf = self._sbuffer
if not buf:
return
# if there is enough data in the buffer, or if we haven't sent in a while, send now:
if not (
self._force_send
or len(buf) >= self.MIN_PACKET_SIZE
or self._last_send + self.WAIT_FOR_BUFFER_GROWTH_SECONDS < time.monotonic()
or self.session.send_size < self.WARMUP_BUDGET_SIZE
):
return
assert buf[-2:] in (b"}\n", b"]\n"), f"unexpected json-rpc terminator: {buf[-2:]=!r}"
# either (1) pad length to next power of two, to create "lsize" packet:
payload_lsize = len(buf)
total_lsize = max(self.MIN_PACKET_SIZE, 2 ** (payload_lsize.bit_length()))
npad_lsize = total_lsize - payload_lsize
# or if that wasted a lot of bandwidth with padding, (2) defer sending some messages
# and create a packet with half that size ("ssize", s for small)
total_ssize = max(self.MIN_PACKET_SIZE, total_lsize // 2)
payload_ssize = buf.rfind(b"\n", 0, total_ssize)
if payload_ssize != -1:
payload_ssize += 1 # for "\n" char
npad_ssize = total_ssize - payload_ssize
else:
npad_ssize = float("inf")
# decide between (1) and (2):
if self._force_send or npad_lsize <= npad_ssize:
# (1) create "lsize" packet: consume full buffer
npad = npad_lsize
p_idx = payload_lsize
else:
# (2) create "ssize" packet: consume some, but defer some for later
npad = npad_ssize
p_idx = payload_ssize
# pad by adding spaces near end
json_rpc_terminator = buf[p_idx-2:p_idx]
assert json_rpc_terminator in (b"}\n", b"]\n"), f"unexpected {json_rpc_terminator=!r}"
buf2 = buf[:p_idx-2] + (npad * b" ") + json_rpc_terminator
self._asyncio_transport.write(buf2)
self._last_send = time.monotonic()
del self._sbuffer[:p_idx]
if not self._sbuffer:
self._sbuffer_has_data_evt.clear()
async def _poll_sbuffer(self):
while not self.is_closing():
await self._can_send.wait()
await self._sbuffer_has_data_evt.wait() # to avoid busy-waiting
self._maybe_consume_sbuffer()
# If there is still data in the buffer, sleep until it would time out.
# note: If the transport is ~idle, when we wake up, we will send the current buf data,
# but if busy, we might wake up to completely new buffer contents. Either is fine.
if len(self._sbuffer) > 0:
timeout_abs = self._last_send + self.WAIT_FOR_BUFFER_GROWTH_SECONDS
timeout_rel = max(0.0, timeout_abs - time.monotonic())
await asyncio.sleep(timeout_rel)
def connection_made(self, transport: asyncio.BaseTransport):
super().connection_made(transport)
coro = self.session.taskgroup.spawn(self._poll_sbuffer())
self._sbuffer_task = self.loop.create_task(coro)
async def close(self, *args, **kwargs):
'''Close the connection and return when closed.'''
# Flush buffer before disconnecting. This makes ReplyAndDisconnect work:
self._force_send = True
self._maybe_consume_sbuffer()
await super().close(*args, **kwargs)