diff --git a/electrum/interface.py b/electrum/interface.py index e47edd024..36cfb13c4 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -49,7 +49,8 @@ import certifi from .util import (ignore_exceptions, log_exceptions, bfh, ESocksProxy, is_integer, is_non_negative_integer, is_hash256_str, is_hex_str, - is_int_or_float, is_non_negative_int_or_float, OldTaskGroup) + is_int_or_float, is_non_negative_int_or_float, OldTaskGroup, + send_exception_to_crash_reporter, error_text_str_to_safe_str) from . import util from . import x509 from . import pem @@ -57,11 +58,13 @@ from . import version from . import blockchain from .blockchain import Blockchain, HEADER_SIZE, CHUNK_SIZE from . import bitcoin +from .bitcoin import DummyAddress, DummyAddressUsedInTxException from . import constants from .i18n import _ from .logging import Logger from .transaction import Transaction from .fee_policy import FEE_ETA_TARGETS +from .lrucache import LRUCache if TYPE_CHECKING: from .network import Network @@ -279,6 +282,34 @@ class InvalidOptionCombination(Exception): pass class ConnectError(NetworkException): pass +class TxBroadcastError(NetworkException): + def get_message_for_gui(self): + raise NotImplementedError() + + +class TxBroadcastHashMismatch(TxBroadcastError): + def get_message_for_gui(self): + return "{}\n{}\n\n{}" \ + .format(_("The server returned an unexpected transaction ID when broadcasting the transaction."), + _("Consider trying to connect to a different server, or updating Electrum."), + str(self)) + + +class TxBroadcastServerReturnedError(TxBroadcastError): + def get_message_for_gui(self): + return "{}\n{}\n\n{}" \ + .format(_("The server returned an error when broadcasting the transaction."), + _("Consider trying to connect to a different server, or updating Electrum."), + str(self)) + + +class TxBroadcastUnknownError(TxBroadcastError): + def get_message_for_gui(self): + return "{}\n{}" \ + .format(_("Unknown error when broadcasting the transaction."), + _("Consider trying to connect to a different server, or updating Electrum.")) + + class _RSClient(RSClient): async def create_connection(self): try: @@ -499,6 +530,7 @@ class Interface(Logger): assert isinstance(server, ServerAddr), f"expected ServerAddr, got {type(server)}" self.ready = network.asyncio_loop.create_future() self.got_disconnected = asyncio.Event() + self._blockchain_updated = asyncio.Event() self.server = server Logger.__init__(self) assert network.config.path @@ -527,6 +559,7 @@ class Interface(Logger): self.tip = 0 self._headers_cache = {} # type: Dict[int, bytes] + self._rawtx_cache = LRUCache(maxsize=20) # type: LRUCache[str, bytes] # txid->rawtx self.fee_estimates_eta = {} # type: Dict[int, int] @@ -1028,6 +1061,8 @@ class Interface(Logger): self.logger.info(f"new chain tip. {height=}") if blockchain_updated: util.trigger_callback('blockchain_updated') + self._blockchain_updated.set() + self._blockchain_updated.clear() util.trigger_callback('network_updated') await self.network.switch_unwanted_fork_interface() await self.network.switch_lagging_interface() @@ -1070,6 +1105,8 @@ class Interface(Logger): continue # report progress to gui/etc util.trigger_callback('blockchain_updated') + self._blockchain_updated.set() + self._blockchain_updated.clear() util.trigger_callback('network_updated') height += num_headers assert height <= next_height+1, (height, self.tip) @@ -1283,6 +1320,8 @@ class Interface(Logger): async def get_transaction(self, tx_hash: str, *, timeout=None) -> str: if not is_hash256_str(tx_hash): raise Exception(f"{repr(tx_hash)} is not a txid") + if rawtx_bytes := self._rawtx_cache.get(tx_hash): + return rawtx_bytes.hex() raw = await self.session.send_request('blockchain.transaction.get', [tx_hash], timeout=timeout) # validate response if not is_hex_str(raw): @@ -1294,8 +1333,40 @@ class Interface(Logger): raise RequestCorrupted(f"cannot deserialize received transaction (txid {tx_hash})") from e if tx.txid() != tx_hash: raise RequestCorrupted(f"received tx does not match expected txid {tx_hash} (got {tx.txid()})") + self._rawtx_cache[tx_hash] = bytes.fromhex(raw) return raw + async def broadcast_transaction(self, tx: 'Transaction', *, timeout=None) -> None: + """caller should handle TxBroadcastError and RequestTimedOut""" + txid_calc = tx.txid() + assert txid_calc is not None + rawtx = tx.serialize() + assert is_hex_str(rawtx) + if timeout is None: + timeout = self.network.get_network_timeout_seconds(NetworkTimeout.Urgent) + if any(DummyAddress.is_dummy_address(txout.address) for txout in tx.outputs()): + raise DummyAddressUsedInTxException("tried to broadcast tx with dummy address!") + try: + out = await self.session.send_request('blockchain.transaction.broadcast', [rawtx], timeout=timeout) + # note: both 'out' and exception messages are untrusted input from the server + except (RequestTimedOut, asyncio.CancelledError, asyncio.TimeoutError): + raise # pass-through + except aiorpcx.jsonrpc.CodeMessageError as e: + self.logger.info(f"broadcast_transaction error [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") + raise TxBroadcastServerReturnedError(sanitize_tx_broadcast_response(e.message)) from e + except BaseException as e: # intentional BaseException for sanity! + self.logger.info(f"broadcast_transaction error2 [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") + send_exception_to_crash_reporter(e) + raise TxBroadcastUnknownError() from e + if out != txid_calc: + self.logger.info(f"unexpected txid for broadcast_transaction [DO NOT TRUST THIS MESSAGE]: " + f"{error_text_str_to_safe_str(out)} != {txid_calc}. tx={str(tx)}") + raise TxBroadcastHashMismatch(_("Server returned unexpected transaction ID.")) + # broadcast succeeded. + # We now cache the rawtx, for *this interface only*. The tx likely touches some ismine addresses, affecting + # the status of a scripthash we are subscribed to. Caching here will save a future get_transaction RPC. + self._rawtx_cache[txid_calc] = bytes.fromhex(rawtx) + async def get_history_for_scripthash(self, sh: str) -> List[dict]: if not is_hash256_str(sh): raise Exception(f"{repr(sh)} is not a scripthash") @@ -1462,6 +1533,190 @@ def _assert_header_does_not_check_against_any_chain(header: dict) -> None: raise Exception('bad_header must not check!') +def sanitize_tx_broadcast_response(server_msg) -> str: + # Unfortunately, bitcoind and hence the Electrum protocol doesn't return a useful error code. + # So, we use substring matching to grok the error message. + # server_msg is untrusted input so it should not be shown to the user. see #4968 + server_msg = str(server_msg) + server_msg = server_msg.replace("\n", r"\n") + + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/script/script_error.cpp + script_error_messages = { + r"Script evaluated without error but finished with a false/empty top stack element", + r"Script failed an OP_VERIFY operation", + r"Script failed an OP_EQUALVERIFY operation", + r"Script failed an OP_CHECKMULTISIGVERIFY operation", + r"Script failed an OP_CHECKSIGVERIFY operation", + r"Script failed an OP_NUMEQUALVERIFY operation", + r"Script is too big", + r"Push value size limit exceeded", + r"Operation limit exceeded", + r"Stack size limit exceeded", + r"Signature count negative or greater than pubkey count", + r"Pubkey count negative or limit exceeded", + r"Opcode missing or not understood", + r"Attempted to use a disabled opcode", + r"Operation not valid with the current stack size", + r"Operation not valid with the current altstack size", + r"OP_RETURN was encountered", + r"Invalid OP_IF construction", + r"Negative locktime", + r"Locktime requirement not satisfied", + r"Signature hash type missing or not understood", + r"Non-canonical DER signature", + r"Data push larger than necessary", + r"Only push operators allowed in signatures", + r"Non-canonical signature: S value is unnecessarily high", + r"Dummy CHECKMULTISIG argument must be zero", + r"OP_IF/NOTIF argument must be minimal", + r"Signature must be zero for failed CHECK(MULTI)SIG operation", + r"NOPx reserved for soft-fork upgrades", + r"Witness version reserved for soft-fork upgrades", + r"Taproot version reserved for soft-fork upgrades", + r"OP_SUCCESSx reserved for soft-fork upgrades", + r"Public key version reserved for soft-fork upgrades", + r"Public key is neither compressed or uncompressed", + r"Stack size must be exactly one after execution", + r"Extra items left on stack after execution", + r"Witness program has incorrect length", + r"Witness program was passed an empty witness", + r"Witness program hash mismatch", + r"Witness requires empty scriptSig", + r"Witness requires only-redeemscript scriptSig", + r"Witness provided for non-witness script", + r"Using non-compressed keys in segwit", + r"Invalid Schnorr signature size", + r"Invalid Schnorr signature hash type", + r"Invalid Schnorr signature", + r"Invalid Taproot control block size", + r"Too much signature validation relative to witness weight", + r"OP_CHECKMULTISIG(VERIFY) is not available in tapscript", + r"OP_IF/NOTIF argument must be minimal in tapscript", + r"Using OP_CODESEPARATOR in non-witness script", + r"Signature is found in scriptCode", + } + for substring in script_error_messages: + if substring in server_msg: + return substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/validation.cpp + # grep "REJECT_" + # grep "TxValidationResult" + # should come after script_error.cpp (due to e.g. "non-mandatory-script-verify-flag") + validation_error_messages = { + r"coinbase": None, + r"tx-size-small": None, + r"non-final": None, + r"txn-already-in-mempool": None, + r"txn-mempool-conflict": None, + r"txn-already-known": None, + r"non-BIP68-final": None, + r"bad-txns-nonstandard-inputs": None, + r"bad-witness-nonstandard": None, + r"bad-txns-too-many-sigops": None, + r"mempool min fee not met": + ("mempool min fee not met\n" + + _("Your transaction is paying a fee that is so low that the bitcoin node cannot " + "fit it into its mempool. The mempool is already full of hundreds of megabytes " + "of transactions that all pay higher fees. Try to increase the fee.")), + r"min relay fee not met": None, + r"absurdly-high-fee": None, + r"max-fee-exceeded": None, + r"too-long-mempool-chain": None, + r"bad-txns-spends-conflicting-tx": None, + r"insufficient fee": ("insufficient fee\n" + + _("Your transaction is trying to replace another one in the mempool but it " + "does not meet the rules to do so. Try to increase the fee.")), + r"too many potential replacements": None, + r"replacement-adds-unconfirmed": None, + r"mempool full": None, + r"non-mandatory-script-verify-flag": None, + r"mandatory-script-verify-flag-failed": None, + r"Transaction check failed": None, + } + for substring in validation_error_messages: + if substring in server_msg: + msg = validation_error_messages[substring] + return msg if msg else substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/rpc/rawtransaction.cpp + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/util/error.cpp + # https://github.com/bitcoin/bitcoin/blob/3f83c744ac28b700090e15b5dda2260724a56f49/src/common/messages.cpp#L126 + # grep "RPC_TRANSACTION" + # grep "RPC_DESERIALIZATION_ERROR" + # grep "TransactionError" + rawtransaction_error_messages = { + r"Missing inputs": None, + r"Inputs missing or spent": None, + r"transaction already in block chain": None, + r"Transaction already in block chain": None, + r"Transaction outputs already in utxo set": None, + r"TX decode failed": None, + r"Peer-to-peer functionality missing or disabled": None, + r"Transaction rejected by AcceptToMemoryPool": None, + r"AcceptToMemoryPool failed": None, + r"Transaction rejected by mempool": None, + r"Mempool internal error": None, + r"Fee exceeds maximum configured by user": None, + r"Unspendable output exceeds maximum configured by user": None, + r"Transaction rejected due to invalid package": None, + } + for substring in rawtransaction_error_messages: + if substring in server_msg: + msg = rawtransaction_error_messages[substring] + return msg if msg else substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/consensus/tx_verify.cpp + # https://github.com/bitcoin/bitcoin/blob/c7ad94428ab6f54661d7a5441e1fdd0ebf034903/src/consensus/tx_check.cpp + # grep "REJECT_" + # grep "TxValidationResult" + tx_verify_error_messages = { + r"bad-txns-vin-empty": None, + r"bad-txns-vout-empty": None, + r"bad-txns-oversize": None, + r"bad-txns-vout-negative": None, + r"bad-txns-vout-toolarge": None, + r"bad-txns-txouttotal-toolarge": None, + r"bad-txns-inputs-duplicate": None, + r"bad-cb-length": None, + r"bad-txns-prevout-null": None, + r"bad-txns-inputs-missingorspent": + ("bad-txns-inputs-missingorspent\n" + + _("You might have a local transaction in your wallet that this transaction " + "builds on top. You need to either broadcast or remove the local tx.")), + r"bad-txns-premature-spend-of-coinbase": None, + r"bad-txns-inputvalues-outofrange": None, + r"bad-txns-in-belowout": None, + r"bad-txns-fee-outofrange": None, + } + for substring in tx_verify_error_messages: + if substring in server_msg: + msg = tx_verify_error_messages[substring] + return msg if msg else substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/policy/policy.cpp + # grep "reason =" + # should come after validation.cpp (due to "tx-size" vs "tx-size-small") + # should come after script_error.cpp (due to e.g. "version") + policy_error_messages = { + r"version": _("Transaction uses non-standard version."), + r"tx-size": _("The transaction was rejected because it is too large (in bytes)."), + r"scriptsig-size": None, + r"scriptsig-not-pushonly": None, + r"scriptpubkey": + ("scriptpubkey\n" + + _("Some of the outputs pay to a non-standard script.")), + r"bare-multisig": None, + r"dust": + (_("Transaction could not be broadcast due to dust outputs.\n" + "Some of the outputs are too small in value, probably lower than 1000 satoshis.\n" + "Check the units, make sure you haven't confused e.g. mBTC and BTC.")), + r"multi-op-return": _("The transaction was rejected because it contains multiple OP_RETURN outputs."), + } + for substring in policy_error_messages: + if substring in server_msg: + msg = policy_error_messages[substring] + return msg if msg else substring + # otherwise: + return _("Unknown error") + + def check_cert(host, cert): try: b = pem.dePem(cert, 'CERTIFICATE') diff --git a/electrum/lrucache.py b/electrum/lrucache.py new file mode 100644 index 000000000..9f81984d7 --- /dev/null +++ b/electrum/lrucache.py @@ -0,0 +1,184 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014-2022 Thomas Kemmer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# ----- +# +# This is a stripped down LRU-cache from the "cachetools" library. +# https://github.com/tkem/cachetools/blob/d991ac71b4eb6394be5ec572b835434081393215/src/cachetools/__init__.py + +import collections +import collections.abc +from typing import TypeVar, Dict + + +class _DefaultSize: + + __slots__ = () + + def __getitem__(self, _): + return 1 + + def __setitem__(self, _, value): + assert value == 1 + + def pop(self, _): + return 1 + + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +class Cache(collections.abc.MutableMapping[_KT, _VT]): + """Mutable mapping to serve as a simple cache or cache base class.""" + + __marker = object() + + __size = _DefaultSize() + + def __init__(self, maxsize: int, getsizeof=None): + if getsizeof: + self.getsizeof = getsizeof + if self.getsizeof is not Cache.getsizeof: + self.__size = dict() + self.__data = dict() # type: Dict[_KT, _VT] + self.__currsize = 0 + self.__maxsize = maxsize + + def __repr__(self): + return "%s(%s, maxsize=%r, currsize=%r)" % ( + self.__class__.__name__, + repr(self.__data), + self.__maxsize, + self.__currsize, + ) + + def __getitem__(self, key: _KT) -> _VT: + try: + return self.__data[key] + except KeyError: + return self.__missing__(key) + + def __setitem__(self, key: _KT, value: _VT) -> None: + maxsize = self.__maxsize + size = self.getsizeof(value) + if size > maxsize: + raise ValueError("value too large") + if key not in self.__data or self.__size[key] < size: + while self.__currsize + size > maxsize: + self.popitem() + if key in self.__data: + diffsize = size - self.__size[key] + else: + diffsize = size + self.__data[key] = value + self.__size[key] = size + self.__currsize += diffsize + + def __delitem__(self, key: _KT) -> None: + size = self.__size.pop(key) + del self.__data[key] + self.__currsize -= size + + def __contains__(self, key: _KT) -> bool: + return key in self.__data + + def __missing__(self, key: _KT): + raise KeyError(key) + + def __iter__(self): + return iter(self.__data) + + def __len__(self): + return len(self.__data) + + def get(self, key: _KT, default: _VT = None) -> _VT | None: + if key in self: + return self[key] + else: + return default + + def pop(self, key: _KT, default=__marker) -> _VT: + if key in self: + value = self[key] + del self[key] + elif default is self.__marker: + raise KeyError(key) + else: + value = default + return value + + def setdefault(self, key: _KT, default: _VT = None) -> _VT | None: + if key in self: + value = self[key] + else: + self[key] = value = default + return value + + @property + def maxsize(self) -> int: + """The maximum size of the cache.""" + return self.__maxsize + + @property + def currsize(self) -> int: + """The current size of the cache.""" + return self.__currsize + + @staticmethod + def getsizeof(value) -> int: + """Return the size of a cache element's value.""" + return 1 + + +class LRUCache(Cache[_KT, _VT]): + """Least Recently Used (LRU) cache implementation.""" + + def __init__(self, maxsize: int, getsizeof=None): + Cache.__init__(self, maxsize, getsizeof) + self.__order = collections.OrderedDict() + + def __getitem__(self, key: _KT, cache_getitem=Cache.__getitem__) -> _VT | None: + value = cache_getitem(self, key) + if key in self: # __missing__ may not store item + self.__update(key) + return value + + def __setitem__(self, key: _KT, value, cache_setitem=Cache.__setitem__) -> None: + cache_setitem(self, key, value) + self.__update(key) + + def __delitem__(self, key: _KT, cache_delitem=Cache.__delitem__) -> None: + cache_delitem(self, key) + del self.__order[key] + + def popitem(self) -> tuple[_KT, _VT]: + """Remove and return the `(key, value)` pair least recently used.""" + try: + key = next(iter(self.__order)) + except StopIteration: + raise KeyError("%s is empty" % type(self).__name__) from None + else: + return (key, self.pop(key)) + + def __update(self, key: _KT) -> None: + try: + self.__order.move_to_end(key) + except KeyError: + self.__order[key] = None diff --git a/electrum/network.py b/electrum/network.py index 462cdaa74..b9a9bb7a7 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -43,10 +43,9 @@ from aiohttp import ClientResponse from . import util from .util import ( - log_exceptions, ignore_exceptions, OldTaskGroup, make_aiohttp_session, send_exception_to_crash_reporter, MyEncoder, + log_exceptions, ignore_exceptions, OldTaskGroup, make_aiohttp_session, MyEncoder, NetworkRetryManager, error_text_str_to_safe_str, detect_tor_socks_proxy ) -from .bitcoin import DummyAddress, DummyAddressUsedInTxException from . import constants from . import blockchain from . import dns_hacks @@ -54,7 +53,7 @@ from .transaction import Transaction from .blockchain import Blockchain from .interface import ( Interface, PREFERRED_NETWORK_PROTOCOL, RequestTimedOut, NetworkTimeout, BUCKET_NAME_OF_ONION_SERVERS, - NetworkException, RequestCorrupted, ServerAddr + NetworkException, RequestCorrupted, ServerAddr, TxBroadcastError, ) from .version import PROTOCOL_VERSION from .i18n import _ @@ -287,34 +286,6 @@ class NetworkParameters(NamedTuple): class BestEffortRequestFailed(NetworkException): pass -class TxBroadcastError(NetworkException): - def get_message_for_gui(self): - raise NotImplementedError() - - -class TxBroadcastHashMismatch(TxBroadcastError): - def get_message_for_gui(self): - return "{}\n{}\n\n{}" \ - .format(_("The server returned an unexpected transaction ID when broadcasting the transaction."), - _("Consider trying to connect to a different server, or updating Electrum."), - str(self)) - - -class TxBroadcastServerReturnedError(TxBroadcastError): - def get_message_for_gui(self): - return "{}\n{}\n\n{}" \ - .format(_("The server returned an error when broadcasting the transaction."), - _("Consider trying to connect to a different server, or updating Electrum."), - str(self)) - - -class TxBroadcastUnknownError(TxBroadcastError): - def get_message_for_gui(self): - return "{}\n{}" \ - .format(_("Unknown error when broadcasting the transaction."), - _("Consider trying to connect to a different server, or updating Electrum.")) - - class UntrustedServerReturnedError(NetworkException): def __init__(self, *, original_exception): self.original_exception = original_exception @@ -1096,26 +1067,7 @@ class Network(Logger, NetworkRetryManager[ServerAddr]): """caller should handle TxBroadcastError""" if self.interface is None: # handled by best_effort_reliable raise RequestTimedOut() - if timeout is None: - timeout = self.get_network_timeout_seconds(NetworkTimeout.Urgent) - if any(DummyAddress.is_dummy_address(txout.address) for txout in tx.outputs()): - raise DummyAddressUsedInTxException("tried to broadcast tx with dummy address!") - try: - out = await self.interface.session.send_request('blockchain.transaction.broadcast', [tx.serialize()], timeout=timeout) - # note: both 'out' and exception messages are untrusted input from the server - except (RequestTimedOut, asyncio.CancelledError, asyncio.TimeoutError): - raise # pass-through - except aiorpcx.jsonrpc.CodeMessageError as e: - self.logger.info(f"broadcast_transaction error [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") - raise TxBroadcastServerReturnedError(self.sanitize_tx_broadcast_response(e.message)) from e - except BaseException as e: # intentional BaseException for sanity! - self.logger.info(f"broadcast_transaction error2 [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") - send_exception_to_crash_reporter(e) - raise TxBroadcastUnknownError() from e - if out != tx.txid(): - self.logger.info(f"unexpected txid for broadcast_transaction [DO NOT TRUST THIS MESSAGE]: " - f"{error_text_str_to_safe_str(out)} != {tx.txid()}. tx={str(tx)}") - raise TxBroadcastHashMismatch(_("Server returned unexpected transaction ID.")) + await self.interface.broadcast_transaction(tx, timeout=timeout) async def try_broadcasting(self, tx, name) -> bool: try: @@ -1127,190 +1079,6 @@ class Network(Logger, NetworkRetryManager[ServerAddr]): self.logger.info(f'success: broadcasting {name} {tx.txid()}') return True - @staticmethod - def sanitize_tx_broadcast_response(server_msg) -> str: - # Unfortunately, bitcoind and hence the Electrum protocol doesn't return a useful error code. - # So, we use substring matching to grok the error message. - # server_msg is untrusted input so it should not be shown to the user. see #4968 - server_msg = str(server_msg) - server_msg = server_msg.replace("\n", r"\n") - - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/script/script_error.cpp - script_error_messages = { - r"Script evaluated without error but finished with a false/empty top stack element", - r"Script failed an OP_VERIFY operation", - r"Script failed an OP_EQUALVERIFY operation", - r"Script failed an OP_CHECKMULTISIGVERIFY operation", - r"Script failed an OP_CHECKSIGVERIFY operation", - r"Script failed an OP_NUMEQUALVERIFY operation", - r"Script is too big", - r"Push value size limit exceeded", - r"Operation limit exceeded", - r"Stack size limit exceeded", - r"Signature count negative or greater than pubkey count", - r"Pubkey count negative or limit exceeded", - r"Opcode missing or not understood", - r"Attempted to use a disabled opcode", - r"Operation not valid with the current stack size", - r"Operation not valid with the current altstack size", - r"OP_RETURN was encountered", - r"Invalid OP_IF construction", - r"Negative locktime", - r"Locktime requirement not satisfied", - r"Signature hash type missing or not understood", - r"Non-canonical DER signature", - r"Data push larger than necessary", - r"Only push operators allowed in signatures", - r"Non-canonical signature: S value is unnecessarily high", - r"Dummy CHECKMULTISIG argument must be zero", - r"OP_IF/NOTIF argument must be minimal", - r"Signature must be zero for failed CHECK(MULTI)SIG operation", - r"NOPx reserved for soft-fork upgrades", - r"Witness version reserved for soft-fork upgrades", - r"Taproot version reserved for soft-fork upgrades", - r"OP_SUCCESSx reserved for soft-fork upgrades", - r"Public key version reserved for soft-fork upgrades", - r"Public key is neither compressed or uncompressed", - r"Stack size must be exactly one after execution", - r"Extra items left on stack after execution", - r"Witness program has incorrect length", - r"Witness program was passed an empty witness", - r"Witness program hash mismatch", - r"Witness requires empty scriptSig", - r"Witness requires only-redeemscript scriptSig", - r"Witness provided for non-witness script", - r"Using non-compressed keys in segwit", - r"Invalid Schnorr signature size", - r"Invalid Schnorr signature hash type", - r"Invalid Schnorr signature", - r"Invalid Taproot control block size", - r"Too much signature validation relative to witness weight", - r"OP_CHECKMULTISIG(VERIFY) is not available in tapscript", - r"OP_IF/NOTIF argument must be minimal in tapscript", - r"Using OP_CODESEPARATOR in non-witness script", - r"Signature is found in scriptCode", - } - for substring in script_error_messages: - if substring in server_msg: - return substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/validation.cpp - # grep "REJECT_" - # grep "TxValidationResult" - # should come after script_error.cpp (due to e.g. "non-mandatory-script-verify-flag") - validation_error_messages = { - r"coinbase": None, - r"tx-size-small": None, - r"non-final": None, - r"txn-already-in-mempool": None, - r"txn-mempool-conflict": None, - r"txn-already-known": None, - r"non-BIP68-final": None, - r"bad-txns-nonstandard-inputs": None, - r"bad-witness-nonstandard": None, - r"bad-txns-too-many-sigops": None, - r"mempool min fee not met": - ("mempool min fee not met\n" + - _("Your transaction is paying a fee that is so low that the bitcoin node cannot " - "fit it into its mempool. The mempool is already full of hundreds of megabytes " - "of transactions that all pay higher fees. Try to increase the fee.")), - r"min relay fee not met": None, - r"absurdly-high-fee": None, - r"max-fee-exceeded": None, - r"too-long-mempool-chain": None, - r"bad-txns-spends-conflicting-tx": None, - r"insufficient fee": ("insufficient fee\n" + - _("Your transaction is trying to replace another one in the mempool but it " - "does not meet the rules to do so. Try to increase the fee.")), - r"too many potential replacements": None, - r"replacement-adds-unconfirmed": None, - r"mempool full": None, - r"non-mandatory-script-verify-flag": None, - r"mandatory-script-verify-flag-failed": None, - r"Transaction check failed": None, - } - for substring in validation_error_messages: - if substring in server_msg: - msg = validation_error_messages[substring] - return msg if msg else substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/rpc/rawtransaction.cpp - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/util/error.cpp - # https://github.com/bitcoin/bitcoin/blob/3f83c744ac28b700090e15b5dda2260724a56f49/src/common/messages.cpp#L126 - # grep "RPC_TRANSACTION" - # grep "RPC_DESERIALIZATION_ERROR" - # grep "TransactionError" - rawtransaction_error_messages = { - r"Missing inputs": None, - r"Inputs missing or spent": None, - r"transaction already in block chain": None, - r"Transaction already in block chain": None, - r"Transaction outputs already in utxo set": None, - r"TX decode failed": None, - r"Peer-to-peer functionality missing or disabled": None, - r"Transaction rejected by AcceptToMemoryPool": None, - r"AcceptToMemoryPool failed": None, - r"Transaction rejected by mempool": None, - r"Mempool internal error": None, - r"Fee exceeds maximum configured by user": None, - r"Unspendable output exceeds maximum configured by user": None, - r"Transaction rejected due to invalid package": None, - } - for substring in rawtransaction_error_messages: - if substring in server_msg: - msg = rawtransaction_error_messages[substring] - return msg if msg else substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/consensus/tx_verify.cpp - # https://github.com/bitcoin/bitcoin/blob/c7ad94428ab6f54661d7a5441e1fdd0ebf034903/src/consensus/tx_check.cpp - # grep "REJECT_" - # grep "TxValidationResult" - tx_verify_error_messages = { - r"bad-txns-vin-empty": None, - r"bad-txns-vout-empty": None, - r"bad-txns-oversize": None, - r"bad-txns-vout-negative": None, - r"bad-txns-vout-toolarge": None, - r"bad-txns-txouttotal-toolarge": None, - r"bad-txns-inputs-duplicate": None, - r"bad-cb-length": None, - r"bad-txns-prevout-null": None, - r"bad-txns-inputs-missingorspent": - ("bad-txns-inputs-missingorspent\n" + - _("You might have a local transaction in your wallet that this transaction " - "builds on top. You need to either broadcast or remove the local tx.")), - r"bad-txns-premature-spend-of-coinbase": None, - r"bad-txns-inputvalues-outofrange": None, - r"bad-txns-in-belowout": None, - r"bad-txns-fee-outofrange": None, - } - for substring in tx_verify_error_messages: - if substring in server_msg: - msg = tx_verify_error_messages[substring] - return msg if msg else substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/policy/policy.cpp - # grep "reason =" - # should come after validation.cpp (due to "tx-size" vs "tx-size-small") - # should come after script_error.cpp (due to e.g. "version") - policy_error_messages = { - r"version": _("Transaction uses non-standard version."), - r"tx-size": _("The transaction was rejected because it is too large (in bytes)."), - r"scriptsig-size": None, - r"scriptsig-not-pushonly": None, - r"scriptpubkey": - ("scriptpubkey\n" + - _("Some of the outputs pay to a non-standard script.")), - r"bare-multisig": None, - r"dust": - (_("Transaction could not be broadcast due to dust outputs.\n" - "Some of the outputs are too small in value, probably lower than 1000 satoshis.\n" - "Check the units, make sure you haven't confused e.g. mBTC and BTC.")), - r"multi-op-return": _("The transaction was rejected because it contains multiple OP_RETURN outputs."), - } - for substring in policy_error_messages: - if substring in server_msg: - msg = policy_error_messages[substring] - return msg if msg else substring - # otherwise: - return _("Unknown error") - @best_effort_reliable @catch_server_exceptions async def get_transaction(self, tx_hash: str, *, timeout=None) -> str: diff --git a/tests/__init__.py b/tests/__init__.py index 2c4e6f038..663dc6080 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -30,6 +30,7 @@ class ElectrumTestCase(unittest.IsolatedAsyncioTestCase, Logger): """Base class for our unit tests.""" TESTNET = False + REGTEST = False TEST_ANCHOR_CHANNELS = False # maxDiff = None # for debugging @@ -43,19 +44,26 @@ class ElectrumTestCase(unittest.IsolatedAsyncioTestCase, Logger): @classmethod def setUpClass(cls): super().setUpClass() - if cls.TESTNET: + assert not (cls.REGTEST and cls.TESTNET), "regtest and testnet are mutually exclusive" + if cls.REGTEST: + constants.BitcoinRegtest.set_as_network() + elif cls.TESTNET: constants.BitcoinTestnet.set_as_network() @classmethod def tearDownClass(cls): super().tearDownClass() - if cls.TESTNET: + if cls.TESTNET or cls.REGTEST: constants.BitcoinMainnet.set_as_network() def setUp(self): - self._test_lock.acquire() + have_lock = self._test_lock.acquire(timeout=0.1) + if not have_lock: + # This can happen when trying to run the tests in parallel, + # or if a prior test raised during `setUp` or `asyncSetUp` and never released the lock. + raise Exception("timed out waiting for test_lock") super().setUp() - self.electrum_path = tempfile.mkdtemp() + self.electrum_path = tempfile.mkdtemp(prefix="electrum-unittest-base-") assert util._asyncio_event_loop is None, "global event loop already set?!" async def asyncSetUp(self): diff --git a/tests/test_interface.py b/tests/test_interface.py index 84e223e09..da4f607cd 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -1,4 +1,16 @@ -from electrum.interface import ServerAddr +import asyncio +import collections + +import aiorpcx +from aiorpcx import RPCError + +import electrum +from electrum.interface import ServerAddr, Interface, PaddedRSTransport +from electrum import util, blockchain +from electrum.util import OldTaskGroup, bfh +from electrum.logging import Logger +from electrum.simple_config import SimpleConfig +from electrum.transaction import Transaction from . import ElectrumTestCase @@ -46,3 +58,187 @@ class TestServerAddr(ElectrumTestCase): ServerAddr(host="2400:6180:0:d1::86b:e001", port=50002, protocol="s").to_friendly_name()) self.assertEqual("[2400:6180:0:d1::86b:e001]:50001:t", ServerAddr(host="2400:6180:0:d1::86b:e001", port=50001, protocol="t").to_friendly_name()) + + +class MockNetwork: + + def __init__(self, *, config: SimpleConfig): + self.config = config + self.asyncio_loop = util.get_asyncio_loop() + self.taskgroup = OldTaskGroup() + blockchain.read_blockchains(self.config) + blockchain.init_headers_file_for_best_chain() + self.proxy = None + self.debug = True + self.bhi_lock = asyncio.Lock() + self.interface = None # type: Interface | None + + async def connection_down(self, interface: Interface): + pass + def get_network_timeout_seconds(self, request_type) -> int: + return 10 + def check_interface_against_healthy_spread_of_connected_servers(self, iface_to_check: Interface) -> bool: + return True + def update_fee_estimates(self, *, fee_est: dict[int, int] = None): + pass + async def switch_unwanted_fork_interface(self): + pass + async def switch_lagging_interface(self): + pass + + +# regtest chain: +BLOCK_HEADERS = { + 0: bfh("0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff7f2002000000"), + 1: bfh("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f186c8dfd970a4545f79916bc1d75c9d00432f57c89209bf3bb115b7612848f509c25f45bffff7f2000000000"), + 2: bfh("00000020686bdfc6a3db73d5d93e8c9663a720a26ecb1ef20eb05af11b36cdbc57c19f7ebf2cbf153013a1c54abaf70e95198fcef2f3059cc6b4d0f7e876808e7d24d11cc825f45bffff7f2000000000"), + 3: bfh("00000020122baa14f3ef54985ae546d1611559e3f487bd2a0f46e8dbb52fbacc9e237972e71019d7feecd9b8596eca9a67032c5f4641b23b5d731dc393e37de7f9c2f299e725f45bffff7f2000000000"), + 4: bfh("00000020f8016f7ef3a17d557afe05d4ea7ab6bde1b2247b7643896c1b63d43a1598b747a3586da94c71753f27c075f57f44faf913c31177a0957bbda42e7699e3a2141aed25f45bffff7f2001000000"), + 5: bfh("000000201d589c6643c1d121d73b0573e5ee58ab575b8fdf16d507e7e915c5fbfbbfd05e7aee1d692d1615c3bdf52c291032144ce9e3b258a473c17c745047f3431ff8e2ee25f45bffff7f2000000000"), + 6: bfh("00000020b833ed46eea01d4c980f59feee44a66aa1162748b6801029565d1466790c405c3a141ce635cbb1cd2b3a4fcdd0a3380517845ba41736c82a79cab535d31128066526f45bffff7f2001000000"), + 7: bfh("00000020abe8e119d1877c9dc0dc502d1a253fb9a67967c57732d2f71ee0280e8381ff0a9690c2fe7c1a4450c74dc908fe94dd96c3b0637d51475e9e06a78e944a0c7fe28126f45bffff7f2000000000"), + 8: bfh("000000202ce41d94eb70e1518bc1f72523f84a903f9705d967481e324876e1f8cf4d3452148be228a4c3f2061bafe7efdfc4a8d5a94759464b9b5c619994d45dfcaf49e1a126f45bffff7f2000000000"), + 9: bfh("00000020552755b6c59f3d51e361d16281842a4e166007799665b5daed86a063dd89857415681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a462555822552221a626f45bffff7f2000000000"), + 10: bfh("00000020a13a491cbefc93cd1bb1938f19957e22a134faf14c7dee951c45533e2c750f239dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548fab26f45bffff7f2000000000"), + 11: bfh("00000020dbf3a9b55dfefbaf8b6e43a89cf833fa2e208bbc0c1c5d76c0d71b9e4a65337803b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe064b026f45bffff7f2002000000"), + 12: bfh("000000203d0932b3b0c78eccb39a595a28ae4a7c966388648d7783fd1305ec8d40d4fe5fd67cb902a7d807cee7676cb543feec3e053aa824d5dfb528d5b94f9760313d9db726f45bffff7f2001000000"), +} + +_active_server_sessions = set() +def _get_active_server_session() -> 'ServerSession': + assert 1 == len(_active_server_sessions), len(_active_server_sessions) + return list(_active_server_sessions)[0] + +class ServerSession(aiorpcx.RPCSession, Logger): + + def __init__(self, *args, **kwargs): + aiorpcx.RPCSession.__init__(self, *args, **kwargs) + Logger.__init__(self) + self.logger.debug(f'connection from {self.remote_address()}') + self.cur_height = 6 # type: int # chain tip + self.txs = { + "bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb": bfh("020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025100ffffffff0200f2052a010000001600140297bde2689a3c79ffe050583b62f86f2d9dae540000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000"), + } # type: dict[str, bytes] + self._method_counts = collections.defaultdict(int) # type: dict[str, int] + _active_server_sessions.add(self) + + async def connection_lost(self): + await super().connection_lost() + self.logger.debug(f'{self.remote_address()} disconnected') + _active_server_sessions.discard(self) + + async def handle_request(self, request): + handlers = { + 'server.version': self._handle_server_version, + 'blockchain.estimatefee': self._handle_estimatefee, + 'blockchain.headers.subscribe': self._handle_headers_subscribe, + 'blockchain.block.header': self._handle_block_header, + 'blockchain.block.headers': self._handle_block_headers, + 'blockchain.transaction.get': self._handle_transaction_get, + 'blockchain.transaction.broadcast': self._handle_transaction_broadcast, + 'server.ping': self._handle_ping, + } + handler = handlers.get(request.method) + self._method_counts[request.method] += 1 + coro = aiorpcx.handler_invocation(handler, request)() + return await coro + + async def _handle_server_version(self, client_name='', protocol_version=None): + return ['best_server_impl/0.1', '1.4'] + + async def _handle_estimatefee(self, number, mode=None): + return 1000 + + async def _handle_headers_subscribe(self): + return {'hex': BLOCK_HEADERS[self.cur_height].hex(), 'height': self.cur_height} + + async def _handle_block_header(self, height): + return BLOCK_HEADERS[height].hex() + + async def _handle_block_headers(self, start_height, count): + assert start_height <= self.cur_height, (start_height, self.cur_height) + last_height = min(start_height+count-1, self.cur_height) # [start_height, last_height] + count = last_height - start_height + 1 + headers = b"".join(BLOCK_HEADERS[idx] for idx in range(start_height, last_height+1)) + return {'hex': headers.hex(), 'count': count, 'max': 2016} + + async def _handle_ping(self): + return None + + async def _handle_transaction_get(self, tx_hash: str, verbose=False): + assert not verbose + rawtx = self.txs.get(tx_hash) + if rawtx is None: + DAEMON_ERROR = 2 + raise RPCError(DAEMON_ERROR, f'daemon error: unknown txid={tx_hash}') + return rawtx.hex() + + async def _handle_transaction_broadcast(self, raw_tx: str): + tx = Transaction(raw_tx) + self.txs[tx.txid()] = bfh(raw_tx) + return tx.txid() + + +class TestInterface(ElectrumTestCase): + REGTEST = True + + def setUp(self): + super().setUp() + self.config = SimpleConfig({'electrum_path': self.electrum_path}) + self._orig_WAIT_FOR_BUFFER_GROWTH_SECONDS = PaddedRSTransport.WAIT_FOR_BUFFER_GROWTH_SECONDS + PaddedRSTransport.WAIT_FOR_BUFFER_GROWTH_SECONDS = 0 + + def tearDown(self): + PaddedRSTransport.WAIT_FOR_BUFFER_GROWTH_SECONDS = self._orig_WAIT_FOR_BUFFER_GROWTH_SECONDS + super().tearDown() + + async def asyncSetUp(self): + await super().asyncSetUp() + self._server: asyncio.base_events.Server = await aiorpcx.serve_rs(ServerSession, "127.0.0.1") + server_socket_addr = self._server.sockets[0].getsockname() + self._server_port = server_socket_addr[1] + self.network = MockNetwork(config=self.config) + + async def asyncTearDown(self): + if self.network.interface: + await self.network.interface.close() + self._server.close() + await self._server.wait_closed() + await super().asyncTearDown() + + async def _start_iface_and_wait_for_sync(self): + interface = Interface(network=self.network, server=ServerAddr(host="127.0.0.1", port=self._server_port, protocol="t")) + self.network.interface = interface + await util.wait_for2(interface.ready, 5) + await interface._blockchain_updated.wait() + return interface + + async def test_client_syncs_headers_to_tip(self): + interface = await self._start_iface_and_wait_for_sync() + self.assertEqual(_get_active_server_session().cur_height, interface.tip) + self.assertFalse(interface.got_disconnected.is_set()) + + async def test_transaction_get(self): + interface = await self._start_iface_and_wait_for_sync() + # try requesting tx unknown to server: + with self.assertRaises(RPCError) as ctx: + await interface.get_transaction("deadbeef"*8) + self.assertTrue("unknown txid" in ctx.exception.message) + # try requesting known tx: + rawtx = await interface.get_transaction("bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb") + self.assertEqual(rawtx, _get_active_server_session().txs["bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb"].hex()) + self.assertEqual(_get_active_server_session()._method_counts["blockchain.transaction.get"], 2) + + async def test_transaction_broadcast(self): + interface = await self._start_iface_and_wait_for_sync() + rawtx1 = "020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025200ffffffff0200f2052a010000001600140297bde2689a3c79ffe050583b62f86f2d9dae540000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000" + tx = Transaction(rawtx1) + # broadcast + await interface.broadcast_transaction(tx) + self.assertEqual(bfh(rawtx1), _get_active_server_session().txs.get(tx.txid())) + # now request tx. + # as we just broadcast this same tx, this will hit the client iface cache, and won't call the server. + self.assertEqual(_get_active_server_session()._method_counts["blockchain.transaction.get"], 0) + rawtx2 = await interface.get_transaction(tx.txid()) + self.assertEqual(rawtx1, rawtx2) + self.assertEqual(_get_active_server_session()._method_counts["blockchain.transaction.get"], 0) diff --git a/tests/test_lntransport.py b/tests/test_lntransport.py index 07bd1fc27..bcad665db 100644 --- a/tests/test_lntransport.py +++ b/tests/test_lntransport.py @@ -102,6 +102,7 @@ class TestLNTransport(ElectrumTestCase): for t in transports: t.close() server.close() + await server.wait_closed() await f()