From dca6adcbf6b55728ec654d86ff7523ab12d3edf8 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Fri, 5 Jul 2024 17:12:29 +0200 Subject: [PATCH 01/22] lnwire: add msg and tlv types for onion_message, including unknown_tags for test vectors --- electrum/lnwire/onion_wire.csv | 52 ++++++++++++++++++++++++++++++++++ electrum/lnwire/peer_wire.csv | 6 ++++ 2 files changed, 58 insertions(+) diff --git a/electrum/lnwire/onion_wire.csv b/electrum/lnwire/onion_wire.csv index 07b10cdd8..1a995dc56 100644 --- a/electrum/lnwire/onion_wire.csv +++ b/electrum/lnwire/onion_wire.csv @@ -7,8 +7,14 @@ tlvdata,payload,short_channel_id,short_channel_id,short_channel_id, tlvtype,payload,payment_data,8 tlvdata,payload,payment_data,payment_secret,byte,32 tlvdata,payload,payment_data,total_msat,tu64, +tlvtype,payload,encrypted_recipient_data,10 +tlvdata,payload,encrypted_recipient_data,encrypted_data,byte,... +tlvtype,payload,current_blinding_point,12 +tlvdata,payload,current_blinding_point,blinding,point, tlvtype,payload,payment_metadata,16 tlvdata,payload,payment_metadata,payment_metadata,byte,... +tlvtype,payload,total_amount_msat,18 +tlvdata,payload,total_amount_msat,total_msat,tu64, tlvtype,payload,invoice_features,66097 tlvdata,payload,invoice_features,invoice_features,u64, tlvtype,payload,outgoing_node_id,66098 @@ -20,6 +26,29 @@ tlvdata,payload,trampoline_onion_packet,version,byte,1 tlvdata,payload,trampoline_onion_packet,public_key,byte,33 tlvdata,payload,trampoline_onion_packet,hops_data,byte,400 tlvdata,payload,trampoline_onion_packet,hmac,byte,32 +tlvtype,encrypted_data_tlv,padding,1 +tlvdata,encrypted_data_tlv,padding,padding,byte,... +tlvtype,encrypted_data_tlv,short_channel_id,2 +tlvdata,encrypted_data_tlv,short_channel_id,short_channel_id,short_channel_id, +tlvtype,encrypted_data_tlv,next_node_id,4 +tlvdata,encrypted_data_tlv,next_node_id,node_id,point, +tlvtype,encrypted_data_tlv,path_id,6 +tlvdata,encrypted_data_tlv,path_id,data,byte,... +tlvtype,encrypted_data_tlv,next_blinding_override,8 +tlvdata,encrypted_data_tlv,next_blinding_override,blinding,point, +tlvtype,encrypted_data_tlv,payment_relay,10 +tlvdata,encrypted_data_tlv,payment_relay,cltv_expiry_delta,u16, +tlvdata,encrypted_data_tlv,payment_relay,fee_proportional_millionths,u32, +tlvdata,encrypted_data_tlv,payment_relay,fee_base_msat,tu32, +tlvtype,encrypted_data_tlv,payment_constraints,12 +tlvdata,encrypted_data_tlv,payment_constraints,max_cltv_expiry,u32, +tlvdata,encrypted_data_tlv,payment_constraints,htlc_minimum_msat,tu64, +tlvtype,encrypted_data_tlv,allowed_features,14 +tlvdata,encrypted_data_tlv,allowed_features,features,byte,... +tlvtype,encrypted_data_tlv,unknown_tag_561,561 +tlvdata,encrypted_data_tlv,unknown_tag_561,data,byte,... +tlvtype,encrypted_data_tlv,unknown_tag_65535,65535 +tlvdata,encrypted_data_tlv,unknown_tag_65535,data,byte,... msgtype,invalid_realm,PERM|1 msgtype,temporary_node_failure,NODE|2 msgtype,permanent_node_failure,PERM|NODE|2 @@ -67,3 +96,26 @@ msgtype,invalid_onion_payload,PERM|22 msgdata,invalid_onion_payload,type,bigsize, msgdata,invalid_onion_payload,offset,u16, msgtype,mpp_timeout,23 +msgtype,invalid_onion_blinding,BADONION|PERM|24 +msgdata,invalid_onion_blinding,sha256_of_onion,sha256, +tlvtype,onionmsg_tlv,message,1 +tlvdata,onionmsg_tlv,message,text,byte,... +tlvtype,onionmsg_tlv,reply_path,2 +tlvdata,onionmsg_tlv,reply_path,path,blinded_path, +tlvtype,onionmsg_tlv,encrypted_recipient_data,4 +tlvdata,onionmsg_tlv,encrypted_recipient_data,encrypted_recipient_data,byte,... +tlvtype,onionmsg_tlv,invoice_request,64 +tlvdata,onionmsg_tlv,invoice_request,invreq,tlv_invoice_request, +tlvtype,onionmsg_tlv,invoice,66 +tlvdata,onionmsg_tlv,invoice,inv,tlv_invoice, +tlvtype,onionmsg_tlv,invoice_error,68 +tlvdata,onionmsg_tlv,invoice_error,inverr,tlv_invoice_error, +subtype,blinded_path +subtypedata,blinded_path,first_node_id,sciddir_or_pubkey, +subtypedata,blinded_path,blinding,point, +subtypedata,blinded_path,num_hops,byte, +subtypedata,blinded_path,path,onionmsg_hop,num_hops +subtype,onionmsg_hop +subtypedata,onionmsg_hop,blinded_node_id,point, +subtypedata,onionmsg_hop,enclen,u16, +subtypedata,onionmsg_hop,encrypted_recipient_data,byte,enclen diff --git a/electrum/lnwire/peer_wire.csv b/electrum/lnwire/peer_wire.csv index a07bc070b..6fb630b98 100644 --- a/electrum/lnwire/peer_wire.csv +++ b/electrum/lnwire/peer_wire.csv @@ -116,6 +116,8 @@ msgdata,update_add_htlc,amount_msat,u64, msgdata,update_add_htlc,payment_hash,sha256, msgdata,update_add_htlc,cltv_expiry,u32, msgdata,update_add_htlc,onion_routing_packet,byte,1366 +tlvtype,update_add_htlc_tlvs,blinding_point,0 +tlvdata,update_add_htlc_tlvs,blinding_point,blinding,point, msgtype,update_fulfill_htlc,130 msgdata,update_fulfill_htlc,channel_id,channel_id, msgdata,update_fulfill_htlc,id,u64, @@ -229,3 +231,7 @@ msgtype,gossip_timestamp_filter,265 msgdata,gossip_timestamp_filter,chain_hash,chain_hash, msgdata,gossip_timestamp_filter,first_timestamp,u32, msgdata,gossip_timestamp_filter,timestamp_range,u32, +msgtype,onion_message,513 +msgdata,onion_message,blinding,point, +msgdata,onion_message,len,u16, +msgdata,onion_message,onion_message_packet,byte,len From 00bba471ff12e7c5a0b8b8a12b4a6b12bb49882c Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Mon, 26 Feb 2024 21:06:20 +0100 Subject: [PATCH 02/22] onion_wire: use unqualified byte,... type for yet undefined nested tlv types --- electrum/lnwire/onion_wire.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/electrum/lnwire/onion_wire.csv b/electrum/lnwire/onion_wire.csv index 1a995dc56..f37f1b564 100644 --- a/electrum/lnwire/onion_wire.csv +++ b/electrum/lnwire/onion_wire.csv @@ -105,11 +105,11 @@ tlvdata,onionmsg_tlv,reply_path,path,blinded_path, tlvtype,onionmsg_tlv,encrypted_recipient_data,4 tlvdata,onionmsg_tlv,encrypted_recipient_data,encrypted_recipient_data,byte,... tlvtype,onionmsg_tlv,invoice_request,64 -tlvdata,onionmsg_tlv,invoice_request,invreq,tlv_invoice_request, +tlvdata,onionmsg_tlv,invoice_request,invoice_request,byte,... tlvtype,onionmsg_tlv,invoice,66 -tlvdata,onionmsg_tlv,invoice,inv,tlv_invoice, +tlvdata,onionmsg_tlv,invoice,invoice,byte,... tlvtype,onionmsg_tlv,invoice_error,68 -tlvdata,onionmsg_tlv,invoice_error,inverr,tlv_invoice_error, +tlvdata,onionmsg_tlv,invoice_error,invoice_error,byte,... subtype,blinded_path subtypedata,blinded_path,first_node_id,sciddir_or_pubkey, subtypedata,blinded_path,blinding,point, From 7c8dfdecbbf8a9ec68eec122bf23b7623516702d Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Tue, 28 May 2024 10:43:46 +0200 Subject: [PATCH 03/22] lnmsg: additional de/serialization support for onion messages - add support for `subtype`/`subtypedata` type declarations - add new primitive type `sciddir_or_pubkey` - better assert message for cardinality errors --- electrum/lnmsg.py | 150 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 143 insertions(+), 7 deletions(-) diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index 1ff50dc54..ec217dc2d 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -147,6 +147,18 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str]) -> U type_len = 33 elif field_type == 'short_channel_id': type_len = 8 + elif field_type == 'sciddir_or_pubkey': + buf = fd.read(1) + if buf[0] in [0, 1]: + type_len = 9 + elif buf[0] in [2, 3]: + type_len = 33 + else: + raise Exception(f"invalid sciddir_or_pubkey, prefix byte not in range 0-3") + buf += fd.read(type_len - 1) + if len(buf) != type_len: + raise UnexpectedEndOfStream() + return buf if count == "...": total_len = -1 # read all @@ -225,6 +237,14 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str], type_len = 33 elif field_type == 'short_channel_id': type_len = 8 + elif field_type == 'sciddir_or_pubkey': + assert isinstance(value, bytes) + if value[0] in [0, 1]: + type_len = 9 # short_channel_id + elif value[0] in [2, 3]: + type_len = 33 # point + else: + raise Exception(f"invalid sciddir_or_pubkey, prefix byte not in range 0-3") total_len = -1 if count != "...": if type_len is None: @@ -299,6 +319,8 @@ class LNSerializer: self.in_tlv_stream_get_record_type_from_name = {} # type: Dict[str, Dict[str, int]] self.in_tlv_stream_get_record_name_from_type = {} # type: Dict[str, Dict[int, str]] + self.subtypes = {} # type: Dict[str, Dict[str, Sequence[str]]] + if for_onion_wire: path = os.path.join(os.path.dirname(__file__), "lnwire", "onion_wire.csv") else: @@ -348,9 +370,112 @@ class LNSerializer: assert tlv_stream_name == row[1] assert tlv_record_name == row[2] self.in_tlv_stream_get_tlv_record_scheme_from_type[tlv_stream_name][tlv_record_type].append(tuple(row)) + elif row[0] == "subtype": + # subtype, + subtypename = row[1] + assert subtypename not in self.subtypes, f"duplicate declaration of subtype {subtypename}" + self.subtypes[subtypename] = {} + elif row[0] == "subtypedata": + # subtypedata,,,,[] + subtypename = row[1] + fieldname = row[2] + assert subtypename in self.subtypes, f"subtypedata definition for subtype {subtypename} declared before subtype" + assert fieldname not in self.subtypes[subtypename], f"duplicate field definition for {fieldname} for subtype {subtypename}" + self.subtypes[subtypename][fieldname] = tuple(row) else: pass # TODO + def _write_complex_field(self, *, fd: io.BytesIO, field_type: str, count: Union[int, str], + value: Union[List[Dict[str, Any]], Dict[str, Any]]) -> None: + assert fd + assert field_type in self.subtypes, f"unknown subtype {field_type}" + + if isinstance(count, int): + assert count >= 0, f"{count!r} must be non-neg int" + elif count == "...": + pass + else: + raise Exception(f"unexpected field count: {count!r}") + if count == 0: + return + + if count == 1: + assert isinstance(value, dict) or isinstance(value, list) + values = [value] if isinstance(value, dict) else value + else: + assert isinstance(value, list), f'{field_type=}, expected value of type list for {count=}' + values = value + + if count == '...': + count = len(values) + else: + assert count == len(values), f'{field_type=}, expected {count} but got {len(values)}' + if count == 0: + return + + for record in values: + for subtypename, row in self.subtypes[field_type].items(): + # subtypedata,,,,[] + subtype_field_name = row[2] + subtype_field_type = row[3] + subtype_field_count_str = row[4] + + subtype_field_count = _resolve_field_count(subtype_field_count_str, + vars_dict=record, + allow_any=True) + + if subtype_field_name not in record: + raise Exception(f'complex field type {field_type} missing element {subtype_field_name}') + + if subtype_field_type in self.subtypes: + self._write_complex_field(fd=fd, + field_type=subtype_field_type, + count=subtype_field_count, + value=record[subtype_field_name]) + else: + _write_field(fd=fd, + field_type=subtype_field_type, + count=subtype_field_count, + value=record[subtype_field_name]) + + def _read_complex_field(self, *, fd: io.BytesIO, field_type: str, count: Union[int, str])\ + -> Union[bytes, List[Dict[str, Any]], Dict[str, Any]]: + assert fd + if isinstance(count, int): + assert count >= 0, f"{count!r} must be non-neg int" + elif count == "...": + pass + else: + raise Exception(f"unexpected field count: {count!r}") + if count == 0: + return b"" + + parsedlist = [] + + while _num_remaining_bytes_to_read(fd): + parsed = {} + for subtypename, row in self.subtypes[field_type].items(): + # subtypedata,,,,[] + subtype_field_name = row[2] + subtype_field_type = row[3] + subtype_field_count_str = row[4] + + subtype_field_count = _resolve_field_count(subtype_field_count_str, + vars_dict=parsed, + allow_any=True) + + if subtype_field_type in self.subtypes: + parsed[subtype_field_name] = self._read_complex_field(fd=fd, + field_type=subtype_field_type, + count=subtype_field_count) + else: + parsed[subtype_field_name] = _read_field(fd=fd, + field_type=subtype_field_type, + count=subtype_field_count) + parsedlist.append(parsed) + + return parsedlist if count == '...' or count > 1 else parsedlist[0] + def write_tlv_stream(self, *, fd: io.BytesIO, tlv_stream_name: str, **kwargs) -> None: scheme_map = self.in_tlv_stream_get_tlv_record_scheme_from_type[tlv_stream_name] for tlv_record_type, scheme in scheme_map.items(): # note: tlv_record_type is monotonically increasing @@ -372,10 +497,16 @@ class LNSerializer: vars_dict=kwargs[tlv_record_name], allow_any=True) field_value = kwargs[tlv_record_name][field_name] - _write_field(fd=tlv_record_fd, - field_type=field_type, - count=field_count, - value=field_value) + if field_type in self.subtypes: + self._write_complex_field(fd=tlv_record_fd, + field_type=field_type, + count=field_count, + value=field_value) + else: + _write_field(fd=tlv_record_fd, + field_type=field_type, + count=field_count, + value=field_value) else: raise Exception(f"unexpected row in scheme: {row!r}") _write_tlv_record(fd=fd, tlv_type=tlv_record_type, tlv_val=tlv_record_fd.getvalue()) @@ -417,9 +548,14 @@ class LNSerializer: vars_dict=parsed[tlv_record_name], allow_any=True) #print(f">> count={field_count}. parsed={parsed}") - parsed[tlv_record_name][field_name] = _read_field(fd=tlv_record_fd, - field_type=field_type, - count=field_count) + if field_type in self.subtypes: + parsed[tlv_record_name][field_name] = self._read_complex_field(fd=tlv_record_fd, + field_type=field_type, + count=field_count) + else: + parsed[tlv_record_name][field_name] = _read_field(fd=tlv_record_fd, + field_type=field_type, + count=field_count) else: raise Exception(f"unexpected row in scheme: {row!r}") if _num_remaining_bytes_to_read(tlv_record_fd) > 0: From 12ffbfc29ed64acbeac80b81dcfae700e02e8780 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Thu, 9 May 2024 12:59:08 +0200 Subject: [PATCH 04/22] lnutil: add onion message feature flag --- electrum/lnutil.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/electrum/lnutil.py b/electrum/lnutil.py index 0cc951b41..edc0e2fee 100644 --- a/electrum/lnutil.py +++ b/electrum/lnutil.py @@ -1431,6 +1431,12 @@ class LnFeatures(IntFlag): _ln_feature_contexts[OPTION_SHUTDOWN_ANYSEGWIT_REQ] = (LNFC.INIT | LNFC.NODE_ANN) _ln_feature_contexts[OPTION_SHUTDOWN_ANYSEGWIT_OPT] = (LNFC.INIT | LNFC.NODE_ANN) + OPTION_ONION_MESSAGE_REQ = 1 << 38 + OPTION_ONION_MESSAGE_OPT = 1 << 39 + + _ln_feature_contexts[OPTION_ONION_MESSAGE_REQ] = (LNFC.INIT | LNFC.NODE_ANN) + _ln_feature_contexts[OPTION_ONION_MESSAGE_OPT] = (LNFC.INIT | LNFC.NODE_ANN) + OPTION_CHANNEL_TYPE_REQ = 1 << 44 OPTION_CHANNEL_TYPE_OPT = 1 << 45 From 7b4180202aeec7a717439ef6d7f4ab61ad97d861 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Fri, 22 Nov 2024 14:46:21 +0100 Subject: [PATCH 05/22] add onion message support --- electrum/commands.py | 73 +- electrum/lnonion.py | 99 ++- electrum/lnpeer.py | 10 +- electrum/lnrouter.py | 19 +- electrum/lnworker.py | 6 + electrum/onion_message.py | 728 ++++++++++++++++++++ tests/blinded-onion-message-onion-test.json | 143 ++++ tests/test_lnpeer.py | 1 + tests/test_lnrouter.py | 107 ++- tests/test_onion_message.py | 234 +++++++ 10 files changed, 1387 insertions(+), 33 deletions(-) create mode 100644 electrum/onion_message.py create mode 100644 tests/blinded-onion-message-onion-test.json create mode 100644 tests/test_onion_message.py diff --git a/electrum/commands.py b/electrum/commands.py index 436c1e99d..a389dd6da 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -22,7 +22,7 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - +import io import sys import datetime import copy @@ -44,9 +44,12 @@ import electrum_ecc as ecc from . import util from . import keystore -from .util import (bfh, format_satoshis, json_decode, json_normalize, - is_hash256_str, is_hex_str, to_bytes, parse_max_spend, to_decimal, - UserFacingException, InvalidPassword) +from .lnmsg import OnionWireSerializer +from .logging import Logger +from .onion_message import create_blinded_path, send_onion_message_to +from .util import (bfh, format_satoshis, json_decode, json_normalize, is_hash256_str, is_hex_str, to_bytes, + parse_max_spend, to_decimal, UserFacingException, InvalidPassword) + from . import bitcoin from .bitcoin import is_address, hash_160, COIN from .bip32 import BIP32Node @@ -175,11 +178,12 @@ def command(s): return decorator -class Commands: +class Commands(Logger): def __init__(self, *, config: 'SimpleConfig', network: 'Network' = None, daemon: 'Daemon' = None, callback=None): + Logger.__init__(self) self.config = config self.daemon = daemon self.network = network @@ -1464,6 +1468,62 @@ class Commands: "source": self.daemon.fx.exchange.name(), } + @command('wnl') + async def send_onion_message(self, node_id_or_blinded_path_hex: str, message: str, wallet: Abstract_Wallet = None): + """ + Send an onion message with onionmsg_tlv.message payload to node_id. + """ + assert wallet + assert wallet.lnworker + assert node_id_or_blinded_path_hex + assert message + + node_id_or_blinded_path = bfh(node_id_or_blinded_path_hex) + assert len(node_id_or_blinded_path) >= 33 + + destination_payload = { + 'message': {'text': message.encode('utf-8')} + } + + try: + await send_onion_message_to(wallet.lnworker, node_id_or_blinded_path, destination_payload) + return {'success': True} + except Exception as e: + msg = str(e) + + return { + 'success': False, + 'msg': msg + } + + @command('wnl') + async def get_blinded_path_via(self, node_id: str, dummy_hops: int = 0, wallet: Abstract_Wallet = None): + """ + Create a blinded path with node_id as introduction point. Introduction point must be direct peer of me. + """ + # TODO: allow introduction_point to not be a direct peer and construct a route + assert wallet + assert node_id + + pubkey = bfh(node_id) + assert len(pubkey) == 33, 'invalid node_id' + + peer = wallet.lnworker.peers[pubkey] + assert peer, 'node_id not a peer' + + path = [pubkey, wallet.lnworker.node_keypair.pubkey] + session_key = os.urandom(32) + blinded_path = create_blinded_path(session_key, path=path, final_recipient_data={}, dummy_hops=dummy_hops) + + with io.BytesIO() as blinded_path_fd: + OnionWireSerializer._write_complex_field(fd=blinded_path_fd, + field_type='blinded_path', + count=1, + value=blinded_path) + encoded_blinded_path = blinded_path_fd.getvalue() + + return encoded_blinded_path.hex() + def eval_bool(x: str) -> bool: if x == 'false': return False @@ -1492,6 +1552,7 @@ param_descriptions = { 'redeem_script': 'redeem script (hexadecimal)', 'lightning_amount': "Amount sent or received in a submarine swap. Set it to 'dryrun' to receive a value", 'onchain_amount': "Amount sent or received in a submarine swap. Set it to 'dryrun' to receive a value", + 'node_id': "Node pubkey in hex format" } command_options = { @@ -1546,6 +1607,7 @@ command_options = { 'from_ccy': (None, "Currency to convert from"), 'to_ccy': (None, "Currency to convert to"), 'public': (None, 'Channel will be announced'), + 'dummy_hops': (None, 'Number of dummy hops to add'), } @@ -1571,6 +1633,7 @@ arg_types = { 'encrypt_file': eval_bool, 'rbf': eval_bool, 'timeout': float, + 'dummy_hops': int, } config_variables = { diff --git a/electrum/lnonion.py b/electrum/lnonion.py index da9961dbb..fa7337295 100644 --- a/electrum/lnonion.py +++ b/electrum/lnonion.py @@ -30,7 +30,7 @@ from enum import IntEnum import electrum_ecc as ecc -from .crypto import sha256, hmac_oneshot, chacha20_encrypt, get_ecdh +from .crypto import sha256, hmac_oneshot, chacha20_encrypt, get_ecdh, chacha20_poly1305_encrypt, chacha20_poly1305_decrypt from .util import profiler, xor_bytes, bfh from .lnutil import (PaymentFailure, NUM_MAX_HOPS_IN_PAYMENT_PATH, NUM_MAX_EDGES_IN_PAYMENT_PATH, ShortChannelID, OnionFailureCodeMetaFlag) @@ -44,20 +44,25 @@ if TYPE_CHECKING: HOPS_DATA_SIZE = 1300 # also sometimes called routingInfoSize in bolt-04 TRAMPOLINE_HOPS_DATA_SIZE = 400 PER_HOP_HMAC_SIZE = 32 - +ONION_MESSAGE_LARGE_SIZE = 32768 class UnsupportedOnionPacketVersion(Exception): pass class InvalidOnionMac(Exception): pass class InvalidOnionPubkey(Exception): pass +class InvalidPayloadSize(Exception): pass class OnionHopsDataSingle: # called HopData in lnd - def __init__(self, *, payload: dict = None): + def __init__(self, *, payload: dict = None, tlv_stream_name: str = 'payload', blind_fields: dict = None): if payload is None: payload = {} self.payload = payload self.hmac = None + self.tlv_stream_name = tlv_stream_name + if blind_fields is None: + blind_fields = {} + self.blind_fields = blind_fields self._raw_bytes_payload = None # used in unit tests def to_bytes(self) -> bytes: @@ -69,7 +74,7 @@ class OnionHopsDataSingle: # called HopData in lnd # adding TLV payload. note: legacy hop data format no longer supported. payload_fd = io.BytesIO() OnionWireSerializer.write_tlv_stream(fd=payload_fd, - tlv_stream_name="payload", + tlv_stream_name=self.tlv_stream_name, **self.payload) payload_bytes = payload_fd.getvalue() with io.BytesIO() as fd: @@ -79,7 +84,7 @@ class OnionHopsDataSingle: # called HopData in lnd return fd.getvalue() @classmethod - def from_fd(cls, fd: io.BytesIO) -> 'OnionHopsDataSingle': + def from_fd(cls, fd: io.BytesIO, *, tlv_stream_name: str = 'payload') -> 'OnionHopsDataSingle': first_byte = fd.read(1) if len(first_byte) == 0: raise Exception(f"unexpected EOF") @@ -95,9 +100,9 @@ class OnionHopsDataSingle: # called HopData in lnd hop_payload = fd.read(hop_payload_length) if hop_payload_length != len(hop_payload): raise Exception(f"unexpected EOF") - ret = OnionHopsDataSingle() + ret = OnionHopsDataSingle(tlv_stream_name=tlv_stream_name) ret.payload = OnionWireSerializer.read_tlv_stream(fd=io.BytesIO(hop_payload), - tlv_stream_name="payload") + tlv_stream_name=tlv_stream_name) ret.hmac = fd.read(PER_HOP_HMAC_SIZE) assert len(ret.hmac) == PER_HOP_HMAC_SIZE return ret @@ -110,7 +115,7 @@ class OnionPacket: def __init__(self, public_key: bytes, hops_data: bytes, hmac: bytes): assert len(public_key) == 33 - assert len(hops_data) in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE] + assert len(hops_data) in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE, ONION_MESSAGE_LARGE_SIZE] assert len(hmac) == PER_HOP_HMAC_SIZE self.version = 0 self.public_key = public_key @@ -127,13 +132,13 @@ class OnionPacket: ret += self.public_key ret += self.hops_data ret += self.hmac - if len(ret) - 66 not in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE]: + if len(ret) - 66 not in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE, ONION_MESSAGE_LARGE_SIZE]: raise Exception('unexpected length {}'.format(len(ret))) return ret @classmethod def from_bytes(cls, b: bytes): - if len(b) - 66 not in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE]: + if len(b) - 66 not in [HOPS_DATA_SIZE, TRAMPOLINE_HOPS_DATA_SIZE, ONION_MESSAGE_LARGE_SIZE]: raise Exception('unexpected length {}'.format(len(b))) version = b[0] if version != 0: @@ -146,27 +151,38 @@ class OnionPacket: def get_bolt04_onion_key(key_type: bytes, secret: bytes) -> bytes: - if key_type not in (b'rho', b'mu', b'um', b'ammag', b'pad'): + if key_type not in (b'rho', b'mu', b'um', b'ammag', b'pad', b'blinded_node_id'): raise Exception('invalid key_type {}'.format(key_type)) key = hmac_oneshot(key_type, msg=secret, digest=hashlib.sha256) return key def get_shared_secrets_along_route(payment_path_pubkeys: Sequence[bytes], - session_key: bytes) -> Sequence[bytes]: + session_key: bytes) -> Tuple[Sequence[bytes], Sequence[bytes]]: num_hops = len(payment_path_pubkeys) hop_shared_secrets = num_hops * [b''] + hop_blinded_node_ids = num_hops * [b''] ephemeral_key = session_key # compute shared key for each hop for i in range(0, num_hops): hop_shared_secrets[i] = get_ecdh(ephemeral_key, payment_path_pubkeys[i]) + hop_blinded_node_ids[i] = get_blinded_node_id(payment_path_pubkeys[i], hop_shared_secrets[i]) ephemeral_pubkey = ecc.ECPrivkey(ephemeral_key).get_public_key_bytes() blinding_factor = sha256(ephemeral_pubkey + hop_shared_secrets[i]) blinding_factor_int = int.from_bytes(blinding_factor, byteorder="big") ephemeral_key_int = int.from_bytes(ephemeral_key, byteorder="big") ephemeral_key_int = ephemeral_key_int * blinding_factor_int % ecc.CURVE_ORDER ephemeral_key = ephemeral_key_int.to_bytes(32, byteorder="big") - return hop_shared_secrets + return hop_shared_secrets, hop_blinded_node_ids + + +def get_blinded_node_id(node_id: bytes, shared_secret: bytes): + # blinded node id + # B(i) = HMAC256("blinded_node_id", ss(i)) * N(i) + ss_bni_hmac = get_bolt04_onion_key(b'blinded_node_id', shared_secret) + ss_bni_hmac_int = int.from_bytes(ss_bni_hmac, byteorder="big") + blinded_node_id = ecc.ECPubkey(node_id) * ss_bni_hmac_int + return blinded_node_id.get_public_key_bytes() def new_onion_packet( @@ -174,14 +190,31 @@ def new_onion_packet( session_key: bytes, hops_data: Sequence[OnionHopsDataSingle], *, - associated_data: bytes, + associated_data: bytes = b'', trampoline: bool = False, + onion_message: bool = False ) -> OnionPacket: num_hops = len(payment_path_pubkeys) assert num_hops == len(hops_data) - hop_shared_secrets = get_shared_secrets_along_route(payment_path_pubkeys, session_key) + hop_shared_secrets, _ = get_shared_secrets_along_route(payment_path_pubkeys, session_key) + + payload_size = 0 + for i in range(num_hops): + # FIXME: serializing here and again below. cache bytes in OnionHopsDataSingle? _raw_bytes_payload? + payload_size += PER_HOP_HMAC_SIZE + len(hops_data[i].to_bytes()) + if trampoline: + data_size = TRAMPOLINE_HOPS_DATA_SIZE + elif onion_message: + if payload_size <= HOPS_DATA_SIZE: + data_size = HOPS_DATA_SIZE + else: + data_size = ONION_MESSAGE_LARGE_SIZE + else: + data_size = HOPS_DATA_SIZE + + if payload_size > data_size: + raise InvalidPayloadSize(f'payload too big for onion packet (max={data_size}, required={payload_size})') - data_size = TRAMPOLINE_HOPS_DATA_SIZE if trampoline else HOPS_DATA_SIZE filler = _generate_filler(b'rho', hops_data, hop_shared_secrets, data_size) next_hmac = bytes(PER_HOP_HMAC_SIZE) @@ -211,6 +244,30 @@ def new_onion_packet( hmac=next_hmac) +def encrypt_onionmsg_data_tlv(*, shared_secret, **kwargs): + rho_key = get_bolt04_onion_key(b'rho', shared_secret) + with io.BytesIO() as encrypted_data_tlv_fd: + OnionWireSerializer.write_tlv_stream( + fd=encrypted_data_tlv_fd, + tlv_stream_name='encrypted_data_tlv', + **kwargs) + encrypted_data_tlv_bytes = encrypted_data_tlv_fd.getvalue() + encrypted_recipient_data = chacha20_poly1305_encrypt( + key=rho_key, nonce=bytes(12), + data=encrypted_data_tlv_bytes) + return encrypted_recipient_data + + +def decrypt_onionmsg_data_tlv(*, shared_secret: bytes, encrypted_recipient_data: bytes) -> dict: + rho_key = get_bolt04_onion_key(b'rho', shared_secret) + recipient_data_bytes = chacha20_poly1305_decrypt(key=rho_key, nonce=bytes(12), data=encrypted_recipient_data) + + with io.BytesIO(recipient_data_bytes) as fd: + recipient_data = OnionWireSerializer.read_tlv_stream(fd=fd, tlv_stream_name='encrypted_data_tlv') + + return recipient_data + + def calc_hops_data_for_payment( route: 'LNPaymentRoute', amount_msat: int, # that final recipient receives @@ -299,9 +356,11 @@ class ProcessedOnionPacket(NamedTuple): # TODO replay protection def process_onion_packet( onion_packet: OnionPacket, - associated_data: bytes, our_onion_private_key: bytes, - is_trampoline=False) -> ProcessedOnionPacket: + *, + associated_data: bytes = b'', + is_trampoline=False, + tlv_stream_name='payload') -> ProcessedOnionPacket: if not ecc.ECPubkey.is_pubkey_bytes(onion_packet.public_key): raise InvalidOnionPubkey() shared_secret = get_ecdh(our_onion_private_key, onion_packet.public_key) @@ -319,7 +378,7 @@ def process_onion_packet( padded_header = onion_packet.hops_data + bytes(data_size) next_hops_data = xor_bytes(padded_header, stream_bytes) next_hops_data_fd = io.BytesIO(next_hops_data) - hop_data = OnionHopsDataSingle.from_fd(next_hops_data_fd) + hop_data = OnionHopsDataSingle.from_fd(next_hops_data_fd, tlv_stream_name=tlv_stream_name) # trampoline trampoline_onion_packet = hop_data.payload.get('trampoline_onion_packet') if trampoline_onion_packet: @@ -427,7 +486,7 @@ def _decode_onion_error(error_packet: bytes, payment_path_pubkeys: Sequence[byte session_key: bytes) -> Tuple[bytes, int]: """Returns the decoded error bytes, and the index of the sender of the error.""" num_hops = len(payment_path_pubkeys) - hop_shared_secrets = get_shared_secrets_along_route(payment_path_pubkeys, session_key) + hop_shared_secrets, _ = get_shared_secrets_along_route(payment_path_pubkeys, session_key) for i in range(num_hops): ammag_key = get_bolt04_onion_key(b'ammag', hop_shared_secrets[i]) um_key = get_bolt04_onion_key(b'um', hop_shared_secrets[i]) diff --git a/electrum/lnpeer.py b/electrum/lnpeer.py index 426cd37a7..afe96b5cb 100644 --- a/electrum/lnpeer.py +++ b/electrum/lnpeer.py @@ -30,8 +30,8 @@ from .bitcoin import make_op_return, DummyAddress from .transaction import PartialTxOutput, match_script_against_template, Sighash from .logging import Logger from .lnrouter import RouteEdge -from .lnonion import (new_onion_packet, OnionFailureCode, calc_hops_data_for_payment, - process_onion_packet, OnionPacket, construct_onion_error, obfuscate_onion_error, OnionRoutingFailure, +from .lnonion import (new_onion_packet, OnionFailureCode, calc_hops_data_for_payment, process_onion_packet, + OnionPacket, construct_onion_error, obfuscate_onion_error, OnionRoutingFailure, ProcessedOnionPacket, UnsupportedOnionPacketVersion, InvalidOnionMac, InvalidOnionPubkey, OnionFailureCodeMetaFlag) from .lnchannel import Channel, RevokeAndAck, RemoteCtnTooFarInFuture, ChannelState, PeerState, ChanCloseOption, CF_ANNOUNCE_CHANNEL @@ -2905,8 +2905,8 @@ class Peer(Logger, EventListener): try: processed_onion = process_onion_packet( onion_packet, - associated_data=payment_hash, our_onion_private_key=self.privkey, + associated_data=payment_hash, is_trampoline=is_trampoline) except UnsupportedOnionPacketVersion: raise OnionRoutingFailure(code=OnionFailureCode.INVALID_ONION_VERSION, data=failure_data) @@ -2922,3 +2922,7 @@ class Peer(Logger, EventListener): if self.network.config.TEST_FAIL_HTLCS_WITH_TEMP_NODE_FAILURE: raise OnionRoutingFailure(code=OnionFailureCode.TEMPORARY_NODE_FAILURE, data=b'') return processed_onion + + def on_onion_message(self, payload): + if hasattr(self.lnworker, 'onion_message_manager'): # only on LNWallet + self.lnworker.onion_message_manager.on_onion_message(payload) diff --git a/electrum/lnrouter.py b/electrum/lnrouter.py index cab174321..94dbe341a 100644 --- a/electrum/lnrouter.py +++ b/electrum/lnrouter.py @@ -25,7 +25,7 @@ import queue from collections import defaultdict -from typing import Sequence, Tuple, Optional, Dict, TYPE_CHECKING, Set +from typing import Sequence, Tuple, Optional, Dict, TYPE_CHECKING, Set, Callable import time import threading from threading import RLock @@ -531,10 +531,17 @@ class LNPathFinder(Logger): invoice_amount_msat: int, my_sending_channels: Dict[ShortChannelID, 'Channel'] = None, private_route_edges: Dict[ShortChannelID, RouteEdge] = None, + node_filter: Optional[Callable[[bytes, NodeInfo], bool]] = None ) -> Dict[bytes, PathEdge]: # note: we don't lock self.channel_db, so while the path finding runs, # the underlying graph could potentially change... (not good but maybe ~OK?) + # if destination is filtered, there is no route + if node_filter: + node_info = self.channel_db.get_node_info_for_node_id(nodeB) + if not node_filter(nodeB, node_info): + return {} + # run Dijkstra # The search is run in the REVERSE direction, from nodeB to nodeA, # to properly calculate compound routing fees. @@ -577,6 +584,10 @@ class LNPathFinder(Logger): if channel_info is None: continue edge_startnode = channel_info.node2_id if channel_info.node1_id == edge_endnode else channel_info.node1_id + if node_filter: + node_info = self.channel_db.get_node_info_for_node_id(edge_startnode) + if not node_filter(edge_startnode, node_info): + continue is_mine = edge_channel_id in my_sending_channels if is_mine: if edge_startnode == nodeA: # payment outgoing, on our channel @@ -617,6 +628,7 @@ class LNPathFinder(Logger): invoice_amount_msat: int, my_sending_channels: Dict[ShortChannelID, 'Channel'] = None, private_route_edges: Dict[ShortChannelID, RouteEdge] = None, + node_filter: Optional[Callable[[bytes, NodeInfo], bool]] = None ) -> Optional[LNPaymentPath]: """Return a path from nodeA to nodeB.""" assert type(nodeA) is bytes @@ -630,7 +642,8 @@ class LNPathFinder(Logger): nodeB=nodeB, invoice_amount_msat=invoice_amount_msat, my_sending_channels=my_sending_channels, - private_route_edges=private_route_edges) + private_route_edges=private_route_edges, + node_filter=node_filter) if nodeA not in previous_hops: return None # no path found @@ -690,7 +703,7 @@ class LNPathFinder(Logger): nodeA: bytes, nodeB: bytes, invoice_amount_msat: int, - path = None, + path: Optional[Sequence[PathEdge]] = None, my_sending_channels: Dict[ShortChannelID, 'Channel'] = None, private_route_edges: Dict[ShortChannelID, RouteEdge] = None, ) -> Optional[LNPaymentRoute]: diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 693d07734..9e0ceecb3 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -48,6 +48,8 @@ from .transaction import ( from .crypto import ( sha256, chacha20_encrypt, chacha20_decrypt, pw_encode_with_version_and_mac, pw_decode_with_version_and_mac ) + +from .onion_message import OnionMessageManager from .lntransport import LNTransport, LNResponderTransport, LNTransportBase, LNPeerAddr, split_host_port, extract_nodeid, ConnStringFormatError from .lnpeer import Peer, LN_P2P_NETWORK_TIMEOUT from .lnaddr import lnencode, LnAddr, lndecode @@ -817,6 +819,7 @@ class LNWallet(LNWorker): self.nostr_keypair = generate_keypair(BIP32Node.from_xkey(xprv), LnKeyFamily.NOSTR_KEY) self.swap_manager = SwapManager(wallet=self.wallet, lnworker=self) + self.onion_message_manager = OnionMessageManager(self) def has_deterministic_node_id(self) -> bool: return bool(self.db.get('lightning_xprv')) @@ -900,6 +903,7 @@ class LNWallet(LNWorker): self.lnwatcher = LNWalletWatcher(self, network) self.swap_manager.start_network(network) self.lnrater = LNRater(self, network) + self.onion_message_manager.start_network(network=network) for chan in self.channels.values(): if chan.need_to_subscribe(): @@ -929,6 +933,8 @@ class LNWallet(LNWorker): self.lnwatcher = None if self.swap_manager and self.swap_manager.network: # may not be present in tests await self.swap_manager.stop() + if self.onion_message_manager: + await self.onion_message_manager.stop() async def wait_for_received_pending_htlcs_to_get_removed(self): assert self.stopping_soon is True diff --git a/electrum/onion_message.py b/electrum/onion_message.py new file mode 100644 index 000000000..1da74a7b2 --- /dev/null +++ b/electrum/onion_message.py @@ -0,0 +1,728 @@ +# Electrum - Lightweight Bitcoin Client +# Copyright (c) 2023-2024 Thomas Voegtlin +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import asyncio +import copy +import io +import os +import queue +import threading +from random import random + +from typing import TYPE_CHECKING, Optional, List, Sequence + +import electrum_ecc as ecc + +from electrum.lnrouter import PathEdge +from electrum.logging import get_logger, Logger +from electrum.crypto import sha256, get_ecdh +from electrum.lnmsg import OnionWireSerializer +from electrum.lnonion import (get_bolt04_onion_key, OnionPacket, process_onion_packet, + OnionHopsDataSingle, decrypt_onionmsg_data_tlv, encrypt_onionmsg_data_tlv, + get_shared_secrets_along_route, new_onion_packet) +from electrum.lnutil import LnFeatures +from electrum.util import OldTaskGroup, now + +if TYPE_CHECKING: + from electrum.lnworker import LNWallet + from electrum.network import Network + from electrum.lnrouter import NodeInfo + from asyncio import Task + +logger = get_logger(__name__) + + +REQUEST_REPLY_TIMEOUT = 30 +REQUEST_REPLY_RETRY_DELAY = 5 +REQUEST_REPLY_PATHS_MAX = 3 +FORWARD_RETRY_TIMEOUT = 4 +FORWARD_RETRY_DELAY = 2 +FORWARD_MAX_QUEUE = 3 + + +def create_blinded_path(session_key: bytes, path: List[bytes], final_recipient_data: dict, *, + hop_extras: Optional[Sequence[dict]] = None, + dummy_hops: Optional[int] = 0) -> dict: + # dummy hops could be inserted anywhere in the path, but for compatibility just add them at the end + # because blinded paths are usually constructed towards ourselves, and we know we can handle dummy hops. + if dummy_hops: + logger.debug(f'adding {dummy_hops} dummy hops at the end') + path += [path[-1]] * dummy_hops + + introduction_point = path[0] + + blinding = ecc.ECPrivkey(session_key).get_public_key_bytes() + + onionmsg_hops = [] + shared_secrets, blinded_node_ids = get_shared_secrets_along_route(path, session_key) + for i, node_id in enumerate(path): + is_non_final_node = i < len(path) - 1 + + if is_non_final_node: + recipient_data = { + # TODO: SHOULD add padding data to ensure all encrypted_data_tlv(i) have the same length + 'next_node_id': {'node_id': path[i+1]} + } + if hop_extras and i < len(hop_extras): # extra hop data for debugging for now + recipient_data.update(hop_extras[i]) + else: + # TODO: SHOULD add padding data to ensure all encrypted_data_tlv(i) have the same length + recipient_data = final_recipient_data + + encrypted_recipient_data = encrypt_onionmsg_data_tlv(shared_secret=shared_secrets[i], **recipient_data) + + hopdata = { + 'blinded_node_id': blinded_node_ids[i], + 'enclen': len(encrypted_recipient_data), + 'encrypted_recipient_data': encrypted_recipient_data + } + onionmsg_hops.append(hopdata) + + blinded_path = { + 'first_node_id': introduction_point, + 'blinding': blinding, + 'num_hops': len(onionmsg_hops), + 'path': onionmsg_hops + } + + return blinded_path + + +def blinding_privkey(privkey, blinding): + shared_secret = get_ecdh(privkey, blinding) + b_hmac = get_bolt04_onion_key(b'blinded_node_id', shared_secret) + b_hmac_int = int.from_bytes(b_hmac, byteorder="big") + + our_privkey_int = int.from_bytes(privkey, byteorder="big") + our_privkey_int = our_privkey_int * b_hmac_int % ecc.CURVE_ORDER + our_privkey = our_privkey_int.to_bytes(32, byteorder="big") + + return our_privkey + + +def is_onion_message_node(node_id: bytes, node_info: Optional['NodeInfo']): + if not node_info: + return False + return LnFeatures(node_info.features).supports(LnFeatures.OPTION_ONION_MESSAGE_OPT) + + +def encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets): + """encrypt unencrypted onionmsg_tlv.encrypted_recipient_data for hops with blind_fields""" + num_hops = len(hops_data) + for i in range(num_hops): + if hops_data[i].tlv_stream_name == 'onionmsg_tlv' and 'encrypted_recipient_data' not in hops_data[i].payload: + # construct encrypted_recipient_data from blind_fields + encrypted_recipient_data = encrypt_onionmsg_data_tlv(shared_secret=hop_shared_secrets[i], **hops_data[i].blind_fields) + hops_data[i].payload['encrypted_recipient_data'] = {'encrypted_recipient_data': encrypted_recipient_data} + + +async def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> List[PathEdge]: + """Constructs a route to the destination node_id, first by starting with peers with existing channels, + and if no route found, opening a direct peer connection if node_id is found with an address in + channel_db.""" + # TODO: is this the proper way to set up my_sending_channels? + my_active_channels = [ + chan for chan in lnwallet.channels.values() if + chan.is_active() and not chan.is_frozen_for_sending()] + my_sending_channels = {chan.short_channel_id: chan for chan in my_active_channels + if chan.short_channel_id is not None} + # strat1: find route to introduction point over existing channel mesh + # NOTE: nodes that are in channel_db but are offline are not removed from the set + if lnwallet.network.path_finder: + if path := lnwallet.network.path_finder.find_path_for_payment( + nodeA=lnwallet.node_keypair.pubkey, + nodeB=node_id, + invoice_amount_msat=10000, # TODO: do this without amount constraints + node_filter=lambda x, y: True if x == lnwallet.node_keypair.pubkey else is_onion_message_node(x, y), + my_sending_channels=my_sending_channels + ): return path + + # strat2: dest node has host:port in channel_db? then open direct peer connection + if lnwallet.channel_db: + if peer_addr := lnwallet.channel_db.get_last_good_address(node_id): + peer = await lnwallet.add_peer(str(peer_addr)) + await peer.initialized + return [PathEdge(short_channel_id=None, start_node=None, end_node=node_id)] + + raise Exception('no path found') + + +async def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: bytes, destination_payload: dict, session_key: bytes = None): + if session_key is None: + session_key = os.urandom(32) + + if len(node_id_or_blinded_path) > 33: # assume blinded path + with io.BytesIO(node_id_or_blinded_path) as blinded_path_fd: + try: + blinded_path = OnionWireSerializer._read_complex_field( + fd=blinded_path_fd, + field_type='blinded_path', + count=1) + logger.debug(f'blinded path: {blinded_path!r}') + except Exception as e: + logger.error(f'e!r') + raise + + introduction_point = blinded_path['first_node_id'] + + hops_data = [] + blinded_node_ids = [] + + if lnwallet.node_keypair.pubkey == introduction_point: + # blinded path introduction point is me + our_blinding = blinded_path['blinding'] + our_payload = blinded_path['path'][0] + remaining_blinded_path = blinded_path['path'][1:] + assert len(remaining_blinded_path) > 0, 'sending to myself?' + + # decrypt + shared_secret = get_ecdh(lnwallet.node_keypair.privkey, our_blinding) + recipient_data = decrypt_onionmsg_data_tlv( + shared_secret=shared_secret, + encrypted_recipient_data=our_payload['encrypted_recipient_data'] + ) + + peer = lnwallet.peers.get(recipient_data['next_node_id']['node_id']) + assert peer, 'next_node_id not a peer' + + # blinding override? + next_blinding_override = recipient_data.get('next_blinding_override') + if next_blinding_override: + next_blinding = next_blinding_override.get('blinding') + else: + # E_i+1=SHA256(E_i||ss_i) * E_i + blinding_factor = sha256(our_blinding + shared_secret) + blinding_factor_int = int.from_bytes(blinding_factor, byteorder="big") + next_public_key_int = ecc.ECPubkey(our_blinding) * blinding_factor_int + next_blinding = next_public_key_int.get_public_key_bytes() + + blinding = next_blinding + + else: + # we need a route to introduction point + remaining_blinded_path = blinded_path['path'] + peer = lnwallet.peers.get(introduction_point) + # if blinded path introduction point is our direct peer, no need to route-find + if peer: + # start of blinded path is our peer + blinding = blinded_path['blinding'] + else: + path = await create_onion_message_route_to(lnwallet, introduction_point) + + # first edge must be to our peer + peer = lnwallet.peers.get(path[0].end_node) + assert peer, 'first hop not a peer' + + # last edge is to introduction point and start of blinded path. remove from route + assert path[-1].end_node == introduction_point, 'last hop in route must be introduction point' + + path = path[:-1] + + if len(path) == 0: + blinding = blinded_path['blinding'] + else: + payment_path_pubkeys = [edge.end_node for edge in path] + hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route( + payment_path_pubkeys, + session_key) + + hops_data = [ + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={'next_node_id': {'node_id': x.end_node}} + ) for x in path[:-1] + ] + + # final hop pre-ip, add next_blinding_override + final_hop_pre_ip = OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={'next_node_id': {'node_id': introduction_point}, + 'next_blinding_override': {'blinding': blinded_path['blinding']}, + } + ) + + hops_data.append(final_hop_pre_ip) + + # encrypt encrypted_data_tlv here + for i in range(len(hops_data)): + encrypted_recipient_data = encrypt_onionmsg_data_tlv(shared_secret=hop_shared_secrets[i], + **hops_data[i].blind_fields) + hops_data[i].payload['encrypted_recipient_data'] = { + 'encrypted_recipient_data': encrypted_recipient_data + } + + blinding = ecc.ECPrivkey(session_key).get_public_key_bytes() + + # append (remaining) blinded path and payload + blinded_path_blinded_ids = [] + for i, onionmsg_hop in enumerate(remaining_blinded_path): + blinded_path_blinded_ids.append(onionmsg_hop.get('blinded_node_id')) + payload = { + 'encrypted_recipient_data': {'encrypted_recipient_data': onionmsg_hop['encrypted_recipient_data']} + } + if i == len(remaining_blinded_path) - 1: # final hop + payload.update(destination_payload) + hop = OnionHopsDataSingle(tlv_stream_name='onionmsg_tlv', payload=payload) + hops_data.append(hop) + + payment_path_pubkeys = blinded_node_ids + blinded_path_blinded_ids + hop_shared_secrets, _ = get_shared_secrets_along_route(payment_path_pubkeys, session_key) + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + packet = new_onion_packet(payment_path_pubkeys, session_key, hops_data) + packet_b = packet.to_bytes() + + else: # node pubkey + pubkey = node_id_or_blinded_path + + if lnwallet.node_keypair.pubkey == pubkey: + raise Exception('cannot send to myself') + + hops_data = [] + peer = lnwallet.peers.get(pubkey) + + if peer: + # destination is our direct peer, no need to route-find + path = [PathEdge(short_channel_id=None, start_node=None, end_node=pubkey)] + else: + path = await create_onion_message_route_to(lnwallet, pubkey) + + # first edge must be to our peer + peer = lnwallet.peers.get(path[0].end_node) + assert peer, 'first hop not a peer' + + hops_data = [ + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={'next_node_id': {'node_id': x.end_node}} + ) for x in path[1:] + ] + + final_hop = OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + payload=destination_payload + ) + + hops_data.append(final_hop) + + payment_path_pubkeys = [edge.end_node for edge in path] + + hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route(payment_path_pubkeys, session_key) + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + packet = new_onion_packet(blinded_node_ids, session_key, hops_data) + packet_b = packet.to_bytes() + + blinding = ecc.ECPrivkey(session_key).get_public_key_bytes() + + peer.send_message( + "onion_message", + blinding=blinding, + len=len(packet_b), + onion_message_packet=packet_b + ) + + +async def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, + max_paths: int = REQUEST_REPLY_PATHS_MAX, + preferred_node_id: bytes = None) -> List[dict]: + # TODO: build longer paths and/or add dummy hops to increase privacy + my_active_channels = [chan for chan in lnwallet.channels.values() if chan.is_active()] + my_onionmsg_channels = [chan for chan in my_active_channels if lnwallet.peers.get(chan.node_id) and + lnwallet.peers.get(chan.node_id).their_features.supports(LnFeatures.OPTION_ONION_MESSAGE_OPT)] + my_onionmsg_peers = [peer for peer in lnwallet.peers.values() if peer.their_features.supports(LnFeatures.OPTION_ONION_MESSAGE_OPT)] + + result = [] + mynodeid = lnwallet.node_keypair.pubkey + mydata = {'path_id': {'data': path_id}} # same used in every path + if len(my_onionmsg_channels): + # randomize list, but prefer preferred_node_id + rchans = sorted(my_onionmsg_channels, key=lambda x: random() if x.node_id != preferred_node_id else 0) + for chan in rchans[:max_paths]: + blinded_path = create_blinded_path(os.urandom(32), [chan.node_id, mynodeid], mydata) + result.append(blinded_path) + elif len(my_onionmsg_peers): + # randomize list, but prefer preferred_node_id + rpeers = sorted(my_onionmsg_peers, key=lambda x: random() if x.pubkey != preferred_node_id else 0) + for peer in rpeers[:max_paths]: + blinded_path = create_blinded_path(os.urandom(32), [peer.pubkey, mynodeid], mydata) + result.append(blinded_path) + + return result + + +class Timeout(Exception): pass + + +class OnionMessageManager(Logger): + """handle state around onion message sends and receives + - association between onion message and their replies + - manage re-send attempts, TODO: iterate through routes (both directions)""" + + def __init__(self, lnwallet: 'LNWallet'): + Logger.__init__(self) + self.network = None # type: Optional['Network'] + self.taskgroup = None # type: OldTaskGroup + self.lnwallet = lnwallet + + self.pending = {} + self.pending_lock = threading.Lock() + self.requestreply_queue = queue.PriorityQueue() + self.requestreply_queue_notempty = asyncio.Event() + self.forwardqueue = queue.PriorityQueue() + self.forwardqueue_notempty = asyncio.Event() + + def start_network(self, *, network: 'Network'): + assert network + assert self.network is None, "already started" + self.network = network + self.taskgroup = OldTaskGroup() + asyncio.run_coroutine_threadsafe(self.main_loop(), self.network.asyncio_loop) + + async def main_loop(self): + self.logger.info("starting taskgroup.") + try: + async with self.taskgroup as group: + await group.spawn(self.process_request_reply_queue()) + await group.spawn(self.process_forward_queue()) + except Exception as e: + self.logger.exception("taskgroup died.") + else: + self.logger.info("taskgroup stopped.") + + async def stop(self): + await self.taskgroup.cancel_remaining() + + async def process_forward_queue(self): + while True: + try: + scheduled, expires, onion_packet, blinding, node_id = self.forwardqueue.get_nowait() + except queue.Empty: + self.logger.debug(f'fwd queue empty') + self.forwardqueue_notempty.clear() + await self.forwardqueue_notempty.wait() + continue + + if expires <= now(): + self.logger.debug(f'fwd expired {node_id=}') + continue + if scheduled > now(): + # return to queue + self.forwardqueue.put_nowait((scheduled, expires, onion_packet, blinding, node_id)) + await asyncio.sleep(1) # sleep here, as the first queue item wasn't due yet + continue + + try: + onion_packet_b = onion_packet.to_bytes() + next_peer = self.lnwallet.peers.get(node_id) + + next_peer.send_message( + "onion_message", + blinding=blinding, + len=len(onion_packet_b), + onion_message_packet=onion_packet_b + ) + except BaseException as e: + self.logger.debug(f'error while sending {node_id=} e={e!r}') + self.forwardqueue.put_nowait((now() + FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) + + def submit_forward(self, *, + onion_packet: OnionPacket, + blinding: bytes, + node_id: bytes): + if self.forwardqueue.qsize() >= FORWARD_MAX_QUEUE: + self.logger.debug('fwd queue full, dropping packet') + return + expires = now() + FORWARD_RETRY_TIMEOUT + queueitem = (now(), expires, onion_packet, blinding, node_id) + self.forwardqueue.put_nowait(queueitem) + self.forwardqueue_notempty.set() + + async def process_request_reply_queue(self): + while True: + try: + scheduled, expires, key = self.requestreply_queue.get_nowait() + except queue.Empty: + self.logger.debug(f'requestreply queue empty') + self.requestreply_queue_notempty.clear() + await self.requestreply_queue_notempty.wait() + continue + + requestreply = self.get_requestreply(key) + if requestreply is None: + self.logger.debug(f'no data for key {key=}') + continue + if requestreply.get('result') is not None: + self.logger.debug(f'has result! {key=}') + continue + if expires <= now(): + self.logger.debug(f'expired {key=}') + self._set_requestreply_result(key, Timeout()) + continue + if scheduled > now(): + # return to queue + self.requestreply_queue.put_nowait((scheduled, expires, key)) + await asyncio.sleep(1) # sleep here, as the first queue item wasn't due yet + continue + + try: + await self._send_pending_requestreply(key) + except BaseException as e: + self.logger.debug(f'error while sending {key=}') + self._set_requestreply_result(key, e) + else: + self.requestreply_queue.put_nowait((now() + REQUEST_REPLY_RETRY_DELAY, expires, key)) + + def get_requestreply(self, key): + with self.pending_lock: + return self.pending.get(key) + + def _set_requestreply_result(self, key, result): + with self.pending_lock: + requestreply = self.pending.get(key) + if requestreply is None: + return + self.pending[key]['result'] = result + requestreply['ev'].set() + + def _remove_requestreply(self, key): + with self.pending_lock: + requestreply = self.pending.get(key) + if requestreply is None: + return + requestreply['ev'].set() + del self.pending[key] + + def submit_requestreply(self, *, + payload: dict, + node_id_or_blinded_path: bytes, + key: bytes = None) -> 'Task': + """Add onion message to queue for sending. Queued onion message payloads + are supplied with a path_id and a reply_path to determine which request + corresponds with arriving replies. + + If caller has provided 'reply_path' in payload, caller should also provide associating key. + + :return: returns awaitable task""" + if not key: + key = os.urandom(8) + assert type(key) is bytes and len(key) >= 8 + + self.logger.debug(f'submit_requestreply {key=} {payload=} {node_id_or_blinded_path=}') + + with self.pending_lock: + if key in self.pending: + raise Exception(f'{key=} already exists!') + self.pending[key] = { + 'ev': asyncio.Event(), + 'payload': payload, + 'node_id_or_blinded_path': node_id_or_blinded_path + } + + # tuple = (when to process, when it expires, key) + expires = now() + REQUEST_REPLY_TIMEOUT + queueitem = (now(), expires, key) + self.requestreply_queue.put_nowait(queueitem) + task = asyncio.create_task(self._requestreply_task(key)) + self.requestreply_queue_notempty.set() + return task + + async def _requestreply_task(self, key): + requestreply = self.get_requestreply(key) + assert requestreply + if requestreply is None: + return + try: + self.logger.debug(f'wait task start {key}') + await requestreply['ev'].wait() + finally: + self.logger.debug(f'wait task end {key}') + + try: + requestreply = self.get_requestreply(key) + assert requestreply + result = requestreply.get('result') + if isinstance(result, Exception): + raise result + return result + finally: + self._remove_requestreply(key) + + async def _send_pending_requestreply(self, key): + """adds reply_path to payload""" + data = self.get_requestreply(key) + payload = data.get('payload') + node_id_or_blinded_path = data.get('node_id_or_blinded_path') + self.logger.debug(f'send_requestreply {key=} {payload=} {node_id_or_blinded_path=}') + + final_payload = copy.deepcopy(payload) + + if 'reply_path' not in final_payload: + # unless explicitly set in payload, generate reply_path here + path_id = self._path_id_from_payload_and_key(payload, key) + reply_paths = await get_blinded_reply_paths(self.lnwallet, path_id, max_paths=1) + if not reply_paths: + raise Exception(f'Could not create a reply_path for {key=}') + + final_payload['reply_path'] = {'path': reply_paths} + + # TODO: we should try alternate paths when retrying, this is currently not done. + # (send_onion_message_to decides path, without knowledge of prev attempts) + await send_onion_message_to(self.lnwallet, node_id_or_blinded_path, final_payload) + + def _path_id_from_payload_and_key(self, payload: dict, key: bytes) -> bytes: + # TODO: construct path_id in such a way that we can determine the request originated from us and is not spoofed + # TODO: use payload to determine prefix? + return b'electrum' + key + + def on_onion_message_received(self, recipient_data, payload): + # we are destination, sanity checks + # - if `encrypted_data_tlv` contains `allowed_features`: + # - MUST ignore the message if: + # - `encrypted_data_tlv.allowed_features.features` contains an unknown feature bit (even if it is odd). + # - the message uses a feature not included in `encrypted_data_tlv.allowed_features.features`. + if 'allowed_features' in recipient_data: + pass # TODO + + # - if `path_id` is set and corresponds to a path the reader has previously published in a `reply_path`: + # - if the onion message is not a reply to that previous onion: + # - MUST ignore the onion message + # TODO: store path_id and lookup here + if 'path_id' not in recipient_data: + # unsolicited onion_message + self.on_onion_message_received_unsolicited(recipient_data, payload) + else: + self.on_onion_message_received_reply(recipient_data, payload) + + def on_onion_message_received_reply(self, recipient_data, payload): + # check if this reply is associated with a known request + correl_data = recipient_data['path_id'].get('data') + if not correl_data[:8] == b'electrum': + self.logger.warning('not a reply to our request (unknown path_id prefix)') + return + key = correl_data[8:] + requestreply = self.get_requestreply(key) + if requestreply is None: + self.logger.warning('not a reply to our request (unknown request)') + return + + self._set_requestreply_result(key, (recipient_data, payload)) + + def on_onion_message_received_unsolicited(self, recipient_data, payload): + self.logger.debug('unsolicited onion_message received') + self.logger.debug(f'payload: {payload!r}') + + # TODO: currently only accepts simple text 'message' payload. + + if 'message' not in payload: + self.logger.error('Unsupported onion message payload') + return + + if 'text' not in payload['message'] or not isinstance(payload['message']['text'], bytes): + self.logger.error('Malformed \'message\' payload') + return + + try: + text = payload['message']['text'].decode('utf-8') + except Exception as e: + self.logger.error(f'Malformed \'message\' payload: {e!r}') + return + + self.logger.info(f'onion message with text received: {text}') + + def on_onion_message_forward(self, recipient_data, onion_packet, blinding, shared_secret): + if recipient_data.get('path_id'): + self.logger.error('cannot forward onion_message, path_id in encrypted_data_tlv') + return + + next_node_id = recipient_data.get('next_node_id') + if not next_node_id: + self.logger.error('cannot forward onion_message, next_node_id missing in encrypted_data_tlv') + return + next_node_id = next_node_id['node_id'] + + is_dummy_hop = False + if next_node_id == self.lnwallet.node_keypair.pubkey: + self.logger.debug('dummy hop') + is_dummy_hop = True + else: + # is next_node one of our peers? + next_peer = self.lnwallet.peers.get(next_node_id) + if not next_peer: + self.logger.info(f'next node {next_node_id.hex()} not a peer, dropping message') + return + + # blinding override? + next_blinding_override = recipient_data.get('next_blinding_override') + if next_blinding_override: + next_blinding = next_blinding_override.get('blinding') + else: + # E_i+1=SHA256(E_i||ss_i) * E_i + blinding_factor = sha256(blinding + shared_secret) + blinding_factor_int = int.from_bytes(blinding_factor, byteorder="big") + next_public_key_int = ecc.ECPubkey(blinding) * blinding_factor_int + next_blinding = next_public_key_int.get_public_key_bytes() + + if is_dummy_hop: + self.process_onion_message_packet(next_blinding, onion_packet) + return + + self.submit_forward(onion_packet=onion_packet, blinding=next_blinding, node_id=next_node_id) + + def on_onion_message(self, payload): + blinding = payload.get('blinding') + if not blinding: + self.logger.error('missing blinding') + return + packet = payload.get('onion_message_packet') + if payload.get('len', 0) != len(packet): + self.logger.error('invalid/missing length') + return + + self.logger.debug('handling onion message') + + onion_packet = OnionPacket.from_bytes(packet) + self.process_onion_message_packet(blinding, onion_packet) + + def process_onion_message_packet(self, blinding: bytes, onion_packet: OnionPacket): + our_privkey = blinding_privkey(self.lnwallet.node_keypair.privkey, blinding) + processed_onion_packet = process_onion_packet(onion_packet, our_privkey, tlv_stream_name='onionmsg_tlv') + payload = processed_onion_packet.hop_data.payload + + self.logger.debug(f'onion peeled: {processed_onion_packet!r}') + + if not processed_onion_packet.are_we_final: + if any([x not in ['encrypted_recipient_data'] for x in payload.keys()]): + self.logger.error('unexpected data in payload') # non-final nodes only encrypted_recipient_data + return + + # decrypt + shared_secret = get_ecdh(self.lnwallet.node_keypair.privkey, blinding) + recipient_data = decrypt_onionmsg_data_tlv( + shared_secret=shared_secret, + encrypted_recipient_data=payload['encrypted_recipient_data']['encrypted_recipient_data'] + ) + + self.logger.debug(f'parsed recipient_data: {recipient_data!r}') + + if processed_onion_packet.are_we_final: + self.on_onion_message_received(recipient_data, payload) + else: + self.on_onion_message_forward(recipient_data, processed_onion_packet.next_packet, blinding, shared_secret) diff --git a/tests/blinded-onion-message-onion-test.json b/tests/blinded-onion-message-onion-test.json new file mode 100644 index 000000000..f66660cc7 --- /dev/null +++ b/tests/blinded-onion-message-onion-test.json @@ -0,0 +1,143 @@ +{ + "comment": "Test vector creating an onionmessage, including joining an existing one", + "generate": { + "comment": "This sections contains test data for Dave's blinded path Bob->Dave; sender has to prepend a hop to Alice to reach Bob", + "session_key": "0303030303030303030303030303030303030303030303030303030303030303", + "hops": [ + { + "alias": "Alice", + "comment": "Alice->Bob: note next_blinding_override to match that give by Dave for Bob", + "blinding_secret": "6363636363636363636363636363636363636363636363636363636363636363", + "tlvs": { + "next_node_id": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c", + "next_blinding_override": "031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f", + "blinding_override_secret": "0101010101010101010101010101010101010101010101010101010101010101" + }, + "encrypted_data_tlv": "04210324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c0821031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f", + "ss": "c04d2a4c518241cb49f2800eea92554cb543f268b4c73f85693541e86d649205", + "HMAC256('blinded_node_id', ss)": "bc5388417c8db33af18ab7ba43f6a5641861f7b0ecb380e501a739af446a7bf4", + "blinded_node_id": "02d1c3d73f8cac67e7c5b6ec517282d5ba0a52b06a29ec92ff01e12decf76003c1", + "E": "031195a8046dcbb8e17034bca630065e7a0982e4e36f6f7e5a8d4554e4846fcd99", + "H(E || ss)": "83377bd6096f82df3a46afec20d68f3f506168f2007f6e86c2dc267417de9e34", + "next_e": "bf3e8999518c0bb6e876abb0ae01d44b9ba211720048099a2ba5a83afd730cad01", + "rho": "6926df9d4522b26ad4330a51e3481208e4816edd9ae4feaf311ea0342eb90c44", + "encrypted_recipient_data": "49531cf38d3280b7f4af6d6461a2b32e3df50acfd35176fc61422a1096eed4dfc3806f29bf74320f712a61c766e7f7caac0c42f86040125fbaeec0c7613202b206dbdd31fda56394367b66a711bfd7d5bedbe20bed1b" + }, + { + "alias": "Bob", + "comment": "Bob->Carol", + "blinding_secret": "0101010101010101010101010101010101010101010101010101010101010101", + "tlvs": { + "next_node_id": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007", + "unknown_tag_561": "123456" + }, + "encrypted_data_tlv": "0421027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007fd023103123456", + "ss": "196f1f3e0be9d65f88463c1ab63e07f41b4e7c0368c28c3e6aa290cc0d22eaed", + "HMAC256('blinded_node_id', ss)": "c331d35827bdd509a02f1e64d48c7f0d7b2603355abbb1a3733c86e50135608e", + "blinded_node_id": "03f1465ca5cf3ec83f16f9343d02e6c24b76993a93e1dea2398f3147a9be893d7a", + "E": "031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f", + "H(E || ss)": "1889a6cf337d9b34f80bb23a91a2ca194e80d7614f0728bdbda153da85e46b69", + "next_e": "f7ab6dca6152f7b6b0c9d7c82d716af063d72d8eef8816dfc51a8ae828fa7dce01", + "rho": "db991242ce366ab44272f38383476669b713513818397a00d4808d41ea979827", + "encrypted_recipient_data": "adf6771d3983b7f543d1b3d7a12b440b2bd3e1b3b8d6ec1023f6dec4f0e7548a6f57f6dbe9573b0a0f24f7c5773a7dd7a7bdb6bd0ee686d759f5" + }, + { + "alias": "Carol", + "comment": "Carol->Dave", + "blinding_secret": "f7ab6dca6152f7b6b0c9d7c82d716af063d72d8eef8816dfc51a8ae828fa7dce", + "tlvs": { + "padding": "0000000000", + "next_node_id": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991" + }, + "encrypted_data_tlv": "010500000000000421032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991", + "ss": "c7b33d74a723e26331a91c15ae5bc77db28a18b801b6bc5cd5bba98418303a9d", + "HMAC256('blinded_node_id', ss)": "a684c7495444a8cc2a6dfdecdf0819f3cdf4e86b81cc14e39825a40872ecefff", + "blinded_node_id": "035dbc0493aa4e7eea369d6a06e8013fd03e66a5eea91c455ed65950c4942b624b", + "E": "02b684babfd400c8dd48b367e9754b8021a3594a34dc94d7101776c7f6a86d0582", + "H(E || ss)": "2d80c5619a5a68d22dd3d784cab584c2718874922735d36cb36a179c10a796ca", + "next_e": "5de52bb427cc148bf23e509fdc18012004202517e80abcfde21612ae408e6cea01", + "rho": "739851e89b61cab34ee9ba7d5f3c342e4adc8b91a72991664026f68a685f0bdc", + "encrypted_recipient_data": "d8903df7a79ac799a0b59f4ba22f6a599fa32e7ff1a8325fc22b88d278ce3e4840af02adfb82d6145a189ba50c2219c9e4351e634d198e0849ac" + }, + { + "alias": "Dave", + "comment": "Dave is final node, hence path_id", + "blinding_secret": "5de52bb427cc148bf23e509fdc18012004202517e80abcfde21612ae408e6cea", + "tlvs": { + "padding": "", + "path_id": "deadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0", + "unknown_tag_65535": "06c1" + }, + "encrypted_data_tlv": "01000620deadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0fdffff0206c1", + "ss": "024955ed0d4ebbfab13498f5d7aacd00bf096c8d9ed0473cdfc96d90053c86b7", + "HMAC256('blinded_node_id', ss)": "3f5612df60f050ac571aeaaf76655e138529bea6d23293ebe15659f2588cd039", + "blinded_node_id": "0237bf019fa0fbecde8b4a1c7b197c9c1c76f9a23d67dd55bb5e42e1f50bb771a6", + "E": "025aaca62db7ce6b46386206ef9930daa32e979a35cb185a41cb951aa7d254b03c", + "H(E || ss)": "db5719e79919d706eab17eebaad64bd691e56476a42f0e26ae60caa9082f56fa", + "next_e": "ae31d2fbbf2f59038542c13287b9b624ea1a212c82be87c137c3d92aa30a185d01", + "rho": "c47cde57edc790df7b9b6bf921aff5e5eee43f738ab8fa9103ef675495f3f50e", + "encrypted_recipient_data": "bdc03f088764c6224c8f939e321bf096f363b2092db381fc8787f891c8e6dc9284991b98d2a63d9f91fe563065366dd406cd8e112cdaaa80d0e6" + } + ] + }, + "route": { + "comment": "The resulting blinded route Alice to Dave.", + "introduction_node_id": "02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619", + "blinding": "031195a8046dcbb8e17034bca630065e7a0982e4e36f6f7e5a8d4554e4846fcd99", + "hops": [ + { + "blinded_node_id": "02d1c3d73f8cac67e7c5b6ec517282d5ba0a52b06a29ec92ff01e12decf76003c1", + "encrypted_recipient_data": "49531cf38d3280b7f4af6d6461a2b32e3df50acfd35176fc61422a1096eed4dfc3806f29bf74320f712a61c766e7f7caac0c42f86040125fbaeec0c7613202b206dbdd31fda56394367b66a711bfd7d5bedbe20bed1b" + }, + { + "blinded_node_id": "03f1465ca5cf3ec83f16f9343d02e6c24b76993a93e1dea2398f3147a9be893d7a", + "encrypted_recipient_data": "adf6771d3983b7f543d1b3d7a12b440b2bd3e1b3b8d6ec1023f6dec4f0e7548a6f57f6dbe9573b0a0f24f7c5773a7dd7a7bdb6bd0ee686d759f5" + }, + { + "blinded_node_id": "035dbc0493aa4e7eea369d6a06e8013fd03e66a5eea91c455ed65950c4942b624b", + "encrypted_recipient_data": "d8903df7a79ac799a0b59f4ba22f6a599fa32e7ff1a8325fc22b88d278ce3e4840af02adfb82d6145a189ba50c2219c9e4351e634d198e0849ac" + }, + { + "blinded_node_id": "0237bf019fa0fbecde8b4a1c7b197c9c1c76f9a23d67dd55bb5e42e1f50bb771a6", + "encrypted_recipient_data": "bdc03f088764c6224c8f939e321bf096f363b2092db381fc8787f891c8e6dc9284991b98d2a63d9f91fe563065366dd406cd8e112cdaaa80d0e6" + } + ] + }, + "onionmessage": { + "comment": "An onion message which sends a 'hello' to Dave", + "unknown_tag_1": "68656c6c6f", + "onion_message_packet": "0002531fe6068134503d2723133227c867ac8fa6c83c537e9a44c3c5bdbdcb1fe33793b828776d70aabbd8cef1a5b52d5a397ae1a20f20435ff6057cd8be339d5aee226660ef73b64afa45dbf2e6e8e26eb96a259b2db5aeecda1ce2e768bbc35d389d7f320ca3d2bd14e2689bef2f5ac0307eaaabc1924eb972c1563d4646ae131accd39da766257ed35ea36e4222527d1db4fa7b2000aab9eafcceed45e28b5560312d4e2299bd8d1e7fe27d10925966c28d497aec400b4630485e82efbabc00550996bdad5d6a9a8c75952f126d14ad2cff91e16198691a7ef2937de83209285f1fb90944b4e46bca7c856a9ce3da10cdf2a7d00dc2bf4f114bc4d3ed67b91cbde558ce9af86dc81fbdc37f8e301b29e23c1466659c62bdbf8cff5d4c20f0fb0851ec72f5e9385dd40fdd2e3ed67ca4517117825665e50a3e26f73c66998daf18e418e8aef9ce2d20da33c3629db2933640e03e7b44c2edf49e9b482db7b475cfd4c617ae1d46d5c24d697846f9f08561eac2b065f9b382501f6eabf07343ed6c602f61eab99cdb52adf63fd44a8db2d3016387ea708fc1c08591e19b4d9984ebe31edbd684c2ea86526dd8c7732b1d8d9117511dc1b643976d356258fce8313b1cb92682f41ab72dedd766f06de375f9edacbcd0ca8c99b865ea2b7952318ea1fd20775a28028b5cf59dece5de14f615b8df254eee63493a5111ea987224bea006d8f1b60d565eef06ac0da194dba2a6d02e79b2f2f34e9ca6e1984a507319d86e9d4fcaeea41b4b9144e0b1826304d4cc1da61cfc5f8b9850697df8adc5e9d6f3acb3219b02764b4909f2b2b22e799fd66c383414a84a7d791b899d4aa663770009eb122f90282c8cb9cda16aba6897edcf9b32951d0080c0f52be3ca011fbec3fb16423deb47744645c3b05fdbd932edf54ba6efd26e65340a8e9b1d1216582e1b30d64524f8ca2d6c5ba63a38f7120a3ed71bed8960bcac2feee2dd41c90be48e3c11ec518eb3d872779e4765a6cc28c6b0fa71ab57ced73ae963cc630edae4258cba2bf25821a6ae049fec2fca28b5dd1bb004d92924b65701b06dcf37f0ccd147a13a03f9bc0f98b7d78fe9058089756931e2cd0e0ed92ec6759d07b248069526c67e9e6ce095118fd3501ba0f858ef030b76c6f6beb11a09317b5ad25343f4b31aef02bc555951bc7791c2c289ecf94d5544dcd6ad3021ed8e8e3db34b2a73e1eedb57b578b068a5401836d6e382110b73690a94328c404af25e85a8d6b808893d1b71af6a31fadd8a8cc6e31ecc0d9ff7e6b91fd03c274a5c1f1ccd25b61150220a3fddb04c91012f5f7a83a5c90deb2470089d6e38cd5914b9c946eca6e9d31bbf8667d36cf87effc3f3ff283c21dd4137bd569fe7cf758feac94053e4baf7338bb592c8b7c291667fadf4a9bf9a2a154a18f612cbc7f851b3f8f2070e0a9d180622ee4f8e81b0ab250d504cef24116a3ff188cc829fcd8610b56343569e8dc997629410d1967ca9dd1d27eec5e01e4375aad16c46faba268524b154850d0d6fe3a76af2c6aa3e97647c51036049ac565370028d6a439a2672b6face56e1b171496c0722cfa22d9da631be359661617c5d5a2d286c5e19db9452c1e21a0107b6400debda2decb0c838f342dd017cdb2dccdf1fe97e3df3f881856b546997a3fed9e279c720145101567dd56be21688fed66bf9759e432a9aa89cbbd225d13cdea4ca05f7a45cfb6a682a3d5b1e18f7e6cf934fae5098108bae9058d05c3387a01d8d02a656d2bfff67e9f46b2d8a6aac28129e52efddf6e552214c3f8a45bc7a912cca9a7fec1d7d06412c6972cb9e3dc518983f56530b8bffe7f92c4b6eb47d4aef59fb513c4653a42de61bc17ad7728e7fc7590ff05a9e991de03f023d0aaf8688ed6170def5091c66576a424ac1cb" + }, + "decrypt": { + "comment": "This section contains the internal values generated by intermediate nodes when decrypting the onion.", + "hops": [ + { + "alias": "Alice", + "privkey": "4141414141414141414141414141414141414141414141414141414141414141", + "onion_message": "0201031195a8046dcbb8e17034bca630065e7a0982e4e36f6f7e5a8d4554e4846fcd9905560002531fe6068134503d2723133227c867ac8fa6c83c537e9a44c3c5bdbdcb1fe33793b828776d70aabbd8cef1a5b52d5a397ae1a20f20435ff6057cd8be339d5aee226660ef73b64afa45dbf2e6e8e26eb96a259b2db5aeecda1ce2e768bbc35d389d7f320ca3d2bd14e2689bef2f5ac0307eaaabc1924eb972c1563d4646ae131accd39da766257ed35ea36e4222527d1db4fa7b2000aab9eafcceed45e28b5560312d4e2299bd8d1e7fe27d10925966c28d497aec400b4630485e82efbabc00550996bdad5d6a9a8c75952f126d14ad2cff91e16198691a7ef2937de83209285f1fb90944b4e46bca7c856a9ce3da10cdf2a7d00dc2bf4f114bc4d3ed67b91cbde558ce9af86dc81fbdc37f8e301b29e23c1466659c62bdbf8cff5d4c20f0fb0851ec72f5e9385dd40fdd2e3ed67ca4517117825665e50a3e26f73c66998daf18e418e8aef9ce2d20da33c3629db2933640e03e7b44c2edf49e9b482db7b475cfd4c617ae1d46d5c24d697846f9f08561eac2b065f9b382501f6eabf07343ed6c602f61eab99cdb52adf63fd44a8db2d3016387ea708fc1c08591e19b4d9984ebe31edbd684c2ea86526dd8c7732b1d8d9117511dc1b643976d356258fce8313b1cb92682f41ab72dedd766f06de375f9edacbcd0ca8c99b865ea2b7952318ea1fd20775a28028b5cf59dece5de14f615b8df254eee63493a5111ea987224bea006d8f1b60d565eef06ac0da194dba2a6d02e79b2f2f34e9ca6e1984a507319d86e9d4fcaeea41b4b9144e0b1826304d4cc1da61cfc5f8b9850697df8adc5e9d6f3acb3219b02764b4909f2b2b22e799fd66c383414a84a7d791b899d4aa663770009eb122f90282c8cb9cda16aba6897edcf9b32951d0080c0f52be3ca011fbec3fb16423deb47744645c3b05fdbd932edf54ba6efd26e65340a8e9b1d1216582e1b30d64524f8ca2d6c5ba63a38f7120a3ed71bed8960bcac2feee2dd41c90be48e3c11ec518eb3d872779e4765a6cc28c6b0fa71ab57ced73ae963cc630edae4258cba2bf25821a6ae049fec2fca28b5dd1bb004d92924b65701b06dcf37f0ccd147a13a03f9bc0f98b7d78fe9058089756931e2cd0e0ed92ec6759d07b248069526c67e9e6ce095118fd3501ba0f858ef030b76c6f6beb11a09317b5ad25343f4b31aef02bc555951bc7791c2c289ecf94d5544dcd6ad3021ed8e8e3db34b2a73e1eedb57b578b068a5401836d6e382110b73690a94328c404af25e85a8d6b808893d1b71af6a31fadd8a8cc6e31ecc0d9ff7e6b91fd03c274a5c1f1ccd25b61150220a3fddb04c91012f5f7a83a5c90deb2470089d6e38cd5914b9c946eca6e9d31bbf8667d36cf87effc3f3ff283c21dd4137bd569fe7cf758feac94053e4baf7338bb592c8b7c291667fadf4a9bf9a2a154a18f612cbc7f851b3f8f2070e0a9d180622ee4f8e81b0ab250d504cef24116a3ff188cc829fcd8610b56343569e8dc997629410d1967ca9dd1d27eec5e01e4375aad16c46faba268524b154850d0d6fe3a76af2c6aa3e97647c51036049ac565370028d6a439a2672b6face56e1b171496c0722cfa22d9da631be359661617c5d5a2d286c5e19db9452c1e21a0107b6400debda2decb0c838f342dd017cdb2dccdf1fe97e3df3f881856b546997a3fed9e279c720145101567dd56be21688fed66bf9759e432a9aa89cbbd225d13cdea4ca05f7a45cfb6a682a3d5b1e18f7e6cf934fae5098108bae9058d05c3387a01d8d02a656d2bfff67e9f46b2d8a6aac28129e52efddf6e552214c3f8a45bc7a912cca9a7fec1d7d06412c6972cb9e3dc518983f56530b8bffe7f92c4b6eb47d4aef59fb513c4653a42de61bc17ad7728e7fc7590ff05a9e991de03f023d0aaf8688ed6170def5091c66576a424ac1cb", + "next_node_id": "0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c" + }, + { + "alias": "Bob", + "privkey": "4242424242424242424242424242424242424242424242424242424242424242", + "onion_message": "0201031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f05560002536d53f93796cad550b6c68662dca41f7e8c221c31022c64dd1a627b2df3982b25eac261e88369cfc66e1e3b6d9829cb3dcd707046e68a7796065202a7904811bf2608c5611cf74c9eb5371c7eb1a4428bb39a041493e2a568ddb0b2482a6cc6711bc6116cef144ebf988073cb18d9dd4ce2d3aa9de91a7dc6d7c6f11a852024626e66b41ba1158055505dff9cb15aa51099f315564d9ee3ed6349665dc3e209eedf9b5805ee4f69d315df44c80e63d0e2efbdab60ec96f44a3447c6a6ddb1efb6aa4e072bde1dab974081646bfddf3b02daa2b83847d74dd336465e76e9b8fecc2b0414045eeedfc39939088a76820177dd1103c99939e659beb07197bab9f714b30ba8dc83738e9a6553a57888aaeda156c68933a2f4ff35e3f81135076b944ed9856acbfee9c61299a5d1763eadd14bf5eaf71304c8e165e590d7ecbcd25f1650bf5b6c2ad1823b2dc9145e168974ecf6a2273c94decff76d94bc6708007a17f22262d63033c184d0166c14f41b225a956271947aae6ce65890ed8f0d09c6ffe05ec02ee8b9de69d7077a0c5adeb813aabcc1ba8975b73ab06ddea5f4db3c23a1de831602de2b83f990d4133871a1a81e53f86393e6a7c3a7b73f0c099fa72afe26c3027bb9412338a19303bd6e6591c04fb4cde9b832b5f41ae199301ea8c303b5cef3aca599454273565de40e1148156d1f97c1aa9e58459ab318304075e034f5b7899c12587b86776a18a1da96b7bcdc22864fccc4c41538ebce92a6f054d53bf46770273a70e75fe0155cd6d2f2e937465b0825ce3123b8c206fac4c30478fa0f08a97ade7216dce11626401374993213636e93545a31f500562130f2feb04089661ad8c34d5a4cbd2e4e426f37cb094c786198a220a2646ecadc38c04c29ee67b19d662c209a7b30bfecc7fe8bf7d274de0605ee5df4db490f6d32234f6af639d3fce38a2801bcf8d51e9c090a6c6932355a83848129a378095b34e71cb8f51152dc035a4fe8e802fec8de221a02ba5afd6765ce570bef912f87357936ea0b90cb2990f56035e89539ec66e8dbd6ed50835158614096990e019c3eba3d7dd6a77147641c6145e8b17552cd5cf7cd163dd40b9eaeba8c78e03a2cd8c0b7997d6f56d35f38983a202b4eb8a54e14945c4de1a6dde46167e11708b7a5ff5cb9c0f7fc12fae49a012aa90bb1995c038130b749c48e6f1ffb732e92086def42af10fbc460d94abeb7b2fa744a5e9a491d62a08452be8cf2fdef573deedc1fe97098bce889f98200b26f9bb99da9aceddda6d793d8e0e44a2601ef4590cfbb5c3d0197aac691e3d31c20fd8e38764962ca34dabeb85df28feabaf6255d4d0df3d814455186a84423182caa87f9673df770432ad8fdfe78d4888632d460d36d2719e8fa8e4b4ca10d817c5d6bc44a8b2affab8c2ba53b8bf4994d63286c2fad6be04c28661162fa1a67065ecda8ba8c13aee4a8039f4f0110e0c0da2366f178d8903e19136dad6df9d8693ce71f3a270f9941de2a93d9b67bc516207ac1687bf6e00b29723c42c7d9c90df9d5e599dbeb7b73add0a6a2b7aba82f98ac93cb6e60494040445229f983a81c34f7f686d166dfc98ec23a6318d4a02a311ac28d655ea4e0f9c3014984f31e621ef003e98c373561d9040893feece2e0fa6cd2dd565e6fbb2773a2407cb2c3273c306cf71f427f2e551c4092e067cf9869f31ac7c6c80dd52d4f85be57a891a41e34be0d564e39b4af6f46b85339254a58b205fb7e10e7d0470ee73622493f28c08962118c23a1198467e72c4ae1cd482144b419247a5895975ea90d135e2a46ef7e5794a1551a447ff0a0d299b66a7f565cd86531f5e7af5408d85d877ce95b1df12b88b7d5954903a5296325ba478ba1e1a9d1f30a2d5052b2e2889bbd64f72c72bc71d8817288a2", + "next_node_id": "027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007" + }, + { + "alias": "Carol", + "privkey": "4343434343434343434343434343434343434343434343434343434343434343", + "onion_message": "020102b684babfd400c8dd48b367e9754b8021a3594a34dc94d7101776c7f6a86d0582055600029a77e8523162efa1f4208f4f2050cd5c386ddb6ce6d36235ea569d217ec52209fb85fdf7dbc4786c373eebdba0ddc184cfbe6da624f610e93f62c70f2c56be1090b926359969f040f932c03f53974db5656233bd60af375517d4323002937d784c2c88a564bcefe5c33d3fc21c26d94dfacab85e2e19685fd2ff4c543650958524439b6da68779459aee5ffc9dc543339acec73ff43be4c44ddcbe1c11d50e2411a67056ba9db7939d780f5a86123fdd3abd6f075f7a1d78ab7daf3a82798b7ec1e9f1345bc0d1e935098497067e2ae5a51ece396fcb3bb30871ad73aee51b2418b39f00c8e8e22be4a24f4b624e09cb0414dd46239de31c7be035f71e8da4f5a94d15b44061f46414d3f355069b5c5b874ba56704eb126148a22ec873407fe118972127e63ff80e682e410f297f23841777cec0517e933eaf49d7e34bd203266b42081b3a5193b51ccd34b41342bc67cf73523b741f5c012ba2572e9dda15fbe131a6ac2ff24dc2a7622d58b9f3553092cfae7fae3c8864d95f97aa49ec8edeff5d9f5782471160ee412d82ff6767030fc63eec6a93219a108cd41433834b26676a39846a944998796c79cd1cc460531b8ded659cedfd8aecefd91944f00476f1496daafb4ea6af3feacac1390ea510709783c2aa81a29de27f8959f6284f4684102b17815667cbb0645396ac7d542b878d90c42a1f7f00c4c4eedb2a22a219f38afadb4f1f562b6e000a94e75cc38f535b43a3c0384ccef127fde254a9033a317701c710b2b881065723486e3f4d3eea5e12f374a41565fe43fa137c1a252c2153dde055bb343344c65ad0529010ece29bbd405effbebfe3ba21382b94a60ac1a5ffa03f521792a67b30773cb42e862a8a02a8bbd41b842e115969c87d1ff1f8c7b5726b9f20772dd57fe6e4ea41f959a2a673ffad8e2f2a472c4c8564f3a5a47568dd75294b1c7180c500f7392a7da231b1fe9e525ea2d7251afe9ca52a17fe54a116cb57baca4f55b9b6de915924d644cba9dade4ccc01939d7935749c008bafc6d3ad01cd72341ce5ddf7a5d7d21cf0465ab7a3233433aef21f9acf2bfcdc5a8cc003adc4d82ac9d72b36eb74e05c9aa6ccf439ac92e6b84a3191f0764dd2a2e0b4cc3baa08782b232ad6ecd3ca6029bc08cc094aef3aebddcaddc30070cb6023a689641de86cfc6341c8817215a4650f844cd2ca60f2f10c6e44cfc5f23912684d4457bf4f599879d30b79bf12ef1ab8d34dddc15672b82e56169d4c770f0a2a7a960b1e8790773f5ff7fce92219808f16d061cc85e053971213676d28fb48925e9232b66533dbd938458eb2cc8358159df7a2a2e4cf87500ede2afb8ce963a845b98978edf26a6948d4932a6b95d022004556d25515fe158092ce9a913b4b4a493281393ca731e8d8e5a3449b9d888fc4e73ffcbb9c6d6d66e88e03cf6e81a0496ede6e4e4172b08c000601993af38f80c7f68c9d5fff9e0e215cff088285bf039ca731744efcb7825a272ca724517736b4890f47e306b200aa2543c363e2c9090bcf3cf56b5b86868a62471c7123a41740392fc1d5ab28da18dca66618e9af7b42b62b23aba907779e73ca03ec60e6ab9e0484b9cae6578e0fddb6386cb3468506bf6420298bf4a690947ab582255551d82487f271101c72e19e54872ab47eae144db66bc2f8194a666a5daec08d12822cb83a61946234f2dfdbd6ca7d8763e6818adee7b401fcdb1ac42f9df1ac5cc5ac131f2869013c8d6cd29d4c4e3d05bccd34ca83366d616296acf854fa05149bfd763a25b9938e96826a037fdcb85545439c76df6beed3bdbd01458f9cf984997cc4f0a7ac3cc3f5e1eeb59c09cadcf5a537f16e444149c8f17d4bdaef16c9fbabc5ef06eb0f0bf3a07a1beddfeacdaf1df5582d6dbd6bb808d6ab31bc22e5d7", + "next_node_id": "032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991" + }, + { + "alias": "Dave", + "privkey": "4444444444444444444444444444444444444444444444444444444444444444", + "onion_message": "0201025aaca62db7ce6b46386206ef9930daa32e979a35cb185a41cb951aa7d254b03c055600025550b2910294fa73bda99b9de9c851be9cbb481e23194a1743033630efba546b86e7d838d0f6e9cc0ed088dbf6889f0dceca3bfc745bd77d013a31311fa932a8bf1d28387d9ff521eabc651dee8f861fed609a68551145a451f017ec44978addeee97a423c08445531da488fd1ddc998e9cdbfcea59517b53fbf1833f0bbe6188dba6ca773a247220ec934010daca9cc185e1ceb136803469baac799e27a0d82abe53dc48a06a55d1f643885cc7894677dd20a4e4152577d1ba74b870b9279f065f9b340cedb3ca13b7df218e853e10ccd1b59c42a2acf93f489e170ee4373d30ab158b60fc20d3ba73a1f8c750951d69fb5b9321b968ddc8114936412346aff802df65516e1c09c51ef19849ff36c0199fd88c8bec301a30fef0c7cb497901c038611303f64e4174b5daf42832aa5586b84d2c9b95f382f4269a5d1bd4be898618dc78dfd451170f72ca16decac5b03e60702112e439cadd104fb3bbb3d5023c9b80823fdcd0a212a7e1aaa6eeb027adc7f8b3723031d135a09a979a4802788bb7861c6cc85501fb91137768b70aeab309b27b885686604ffc387004ac4f8c44b101c39bc0597ef7fd957f53fc5051f534b10eb3852100962b5e58254e5558689913c26ad6072ea41f5c5db10077cfc91101d4ae393be274c74297da5cc381cd88d54753aaa7df74b2f9da8d88a72bc9218fcd1f19e4ff4aace182312b9509c5175b6988f044c5756d232af02a451a02ca752f3c52747773acff6fd07d2032e6ce562a2c42105d106eba02d0b1904182cdc8c74875b082d4989d3a7e9f0e73de7c75d357f4af976c28c0b206c5e8123fc2391d078592d0d5ff686fd245c0a2de2e535b7cca99c0a37d432a8657393a9e3ca53eec1692159046ba52cb9bc97107349d8673f74cbc97e231f1108005c8d03e24ca813cea2294b39a7a493bcc062708f1f6cf0074e387e7d50e0666ce784ef4d31cb860f6cad767438d9ea5156ff0ae86e029e0247bf94df75ee0cda4f2006061455cb2eaff513d558863ae334cef7a3d45f55e7cc13153c6719e9901c1d4db6c03f643b69ea4860690305651794284d9e61eb848ccdf5a77794d376f0af62e46d4835acce6fd9eef5df73ebb8ea3bb48629766967f446e744ecc57ff3642c4aa1ccee9a2f72d5caa75fa05787d08b79408fce792485fdecdc25df34820fb061275d70b84ece540b0fc47b2453612be34f2b78133a64e812598fbe225fd85415f8ffe5340ce955b5fd9d67dd88c1c531dde298ed25f96df271558c812c26fa386966c76f03a6ebccbca49ac955916929bd42e134f982dde03f924c464be5fd1ba44f8dc4c3cbc8162755fd1d8f7dc044b15b1a796c53df7d8769bb167b2045b49cc71e08908796c92c16a235717cabc4bb9f60f8f66ff4fff1f9836388a99583acebdff4a7fb20f48eedcd1f4bdcc06ec8b48e35307df51d9bc81d38a94992dd135b30079e1f592da6e98dff496cb1a7776460a26b06395b176f585636ebdf7eab692b227a31d6979f5a6141292698e91346b6c806b90c7c6971e481559cae92ee8f4136f2226861f5c39ddd29bbdb118a35dece03f49a96804caea79a3dacfbf09d65f2611b5622de51d98e18151acb3bb84c09caaa0cc80edfa743a4679f37d6167618ce99e73362fa6f213409931762618a61f1738c071bba5afc1db24fe94afb70c40d731908ab9a505f76f57a7d40e708fd3df0efc5b7cbb2a7b75cd23449e09684a2f0e2bfa0d6176c35f96fe94d92fc9fa4103972781f81cb6e8df7dbeb0fc529c600d768bed3f08828b773d284f69e9a203459d88c12d6df7a75be2455fec128f07a497a2b2bf626cc6272d0419ca663e9dc66b8224227eb796f0246dcae9c5b0b6cfdbbd40c3245a610481c92047c968c9fc92c04b89cc41a0c15355a8f", + "tlvs": { + "unknown_tag_1": "68656c6c6f", + "encrypted_recipient_data": "bdc03f088764c6224c8f939e321bf096f363b2092db381fc8787f891c8e6dc9284991b98d2a63d9f91fe563065366dd406cd8e112cdaaa80d0e6" + } + } + ] + } +} diff --git a/tests/test_lnpeer.py b/tests/test_lnpeer.py index 2ad720332..43855f1c0 100644 --- a/tests/test_lnpeer.py +++ b/tests/test_lnpeer.py @@ -164,6 +164,7 @@ class MockLNWallet(Logger, EventListener, NetworkRetryManager[LNPeerAddr]): self.taskgroup = OldTaskGroup() self.lnwatcher = None self.swap_manager = None + self.onion_message_manager = None self.listen_server = None self._channels = {chan.channel_id: chan for chan in chans} self.payment_info = {} diff --git a/tests/test_lnrouter.py b/tests/test_lnrouter.py index 8187934a2..16c3980c6 100644 --- a/tests/test_lnrouter.py +++ b/tests/test_lnrouter.py @@ -6,8 +6,10 @@ import asyncio from typing import Optional from electrum import util +from electrum.channel_db import NodeInfo +from electrum.onion_message import is_onion_message_node from electrum.util import bfh -from electrum.lnutil import ShortChannelID +from electrum.lnutil import ShortChannelID, LnFeatures from electrum.lnonion import (OnionHopsDataSingle, new_onion_packet, process_onion_packet, _decode_onion_error, decode_onion_error, OnionFailureCode, OnionPacket) @@ -28,6 +30,17 @@ def node(character: str) -> bytes: return b'\x02' + f'{character}'.encode() * 32 +def alias(character: str) -> bytes: + return (character * 8).encode('utf-8') + + +def node_features(extra: LnFeatures = None) -> bytes: + lnf = LnFeatures(0) | LnFeatures.VAR_ONION_OPT + if extra: + lnf |= extra + return lnf.to_bytes(8, 'big') + + class Test_LNRouter(ElectrumTestCase): TESTNET = True @@ -63,12 +76,14 @@ class Test_LNRouter(ElectrumTestCase): A -6-> D -4-> C -1-> B -2-> E A -3-> B -1-> C -4-> D -5-> E """ + class fake_network: config = self.config asyncio_loop = util.get_asyncio_loop() trigger_callback = lambda *args: None register_callback = lambda *args: None interface = None + fake_network.channel_db = lnrouter.ChannelDB(fake_network()) fake_network.channel_db.data_loaded.set() self.cdb = fake_network.channel_db @@ -124,8 +139,46 @@ class Test_LNRouter(ElectrumTestCase): 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'len': 0, 'features': b'' }, trusted=True) + + self.cdb.add_node_announcements({ + 'node_id': node('a'), + 'alias': alias('a'), + 'addresses': [], + 'features': node_features(LnFeatures.OPTION_ONION_MESSAGE_OPT), + 'timestamp': 0 + }) + self.cdb.add_node_announcements({ + 'node_id': node('b'), + 'alias': alias('b'), + 'addresses': [], + 'features': node_features(), + 'timestamp': 0 + }) + self.cdb.add_node_announcements({ + 'node_id': node('c'), + 'alias': alias('c'), + 'addresses': [], + 'features': node_features(LnFeatures.OPTION_ONION_MESSAGE_OPT), + 'timestamp': 0 + }) + self.cdb.add_node_announcements({ + 'node_id': node('d'), + 'alias': alias('d'), + 'addresses': [], + 'features': node_features(LnFeatures.OPTION_ONION_MESSAGE_OPT), + 'timestamp': 0 + }) + self.cdb.add_node_announcements({ + 'node_id': node('e'), + 'alias': alias('e'), + 'addresses': [], + 'features': node_features(), + 'timestamp': 0 + }) + def add_chan_upd(payload): self.cdb.add_channel_update(payload, verify=False) + add_chan_upd({'short_channel_id': channel(1), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) add_chan_upd({'short_channel_id': channel(1), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) add_chan_upd({'short_channel_id': channel(2), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0}) @@ -158,6 +211,27 @@ class Test_LNRouter(ElectrumTestCase): self.assertEqual(node('b'), route[0].node_id) self.assertEqual(channel(3), route[0].short_channel_id) + async def test_find_path_for_payment_with_node_filter(self): + self.prepare_graph() + amount_to_send = 100000 + + def node_filter(node_id: bytes, node_info: 'NodeInfo'): + return node_info.node_id != node('b') + + path = self.path_finder.find_path_for_payment( + nodeA=node('a'), + nodeB=node('e'), + invoice_amount_msat=amount_to_send, + node_filter=node_filter) + self.assertEqual([ + PathEdge(start_node=node('a'), end_node=node('d'), short_channel_id=channel(6)), + PathEdge(start_node=node('d'), end_node=node('e'), short_channel_id=channel(5)), + ], path) + + route = self.path_finder.create_route_from_path(path) + self.assertEqual(node('d'), route[0].node_id) + self.assertEqual(channel(6), route[0].short_channel_id) + async def test_find_path_liquidity_hints(self): self.prepare_graph() amount_to_send = 100000 @@ -372,7 +446,7 @@ class Test_LNRouter(ElectrumTestCase): self.assertEqual(bfh('0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619f7f3416a5aa36dc7eeb3ec6d421e9615471ab858ba970cd3cceb768b44e692be2f390c0b7fe70122abae84d7801db070dfb1638cd8d263072206dbed0234f6505e21e282abd8587124c572aad8de04610a136d6c71a7648c0ef66f1b3655d8a9eea1f92349132c93befbd6c37dbfc55615814ae09e4cbef721c01b487007811bbbfdc1fc7bd869aeb70eb08b4140ff5f501394b3653ada2a3b36a263535ea421d26818afb278df46abcec093305b715cac22b0b03645f8f4797cf2987b1bf4bfdd9ed8648ed42ed1a831fc36ccd45416a132580281ddac4e7470e4d2afd675baad9282ec6335403a73e1391427e330996c834db93848b4ae29dd975f678b2f5155ad6865ca23190725d4b7238fb44f0e3762dd59091b45c97d45df8164a15d9ca0329ec76f957b0a0e49ae372154620708df5c0fa991f0dd12b6bff1ebaf9e2376bb64bc24713f7c57da569bcd9c43a50c088416564b786a87d1f40936a051a3dbfe023bd867a5e66148b61cdd24a79f8c18682150e55aa6969ce9becf51f7c69e72deafcd0659f6be4f78463eaef8716e56615c77b3fbea8190806359909dcbec13c1592523b3d2985ec3e83d42cb7286a66a22f58704ddf6979ceb6883ab4ad8ac99d30251035189ffd514e03ce1576844513d66965d4adfc2523f4eee0dede229ab96303e31348c72bc0c8c816c666a904e5ccbabadf5a919720438f4a14dbd4a802f8d4b942f0ca8572f59644c9ac1912c8c8efefc4afa7f19e27411d46b7541c55985e28ce5cd7620b335fea51de55fa00ef977e8522181ad19e5e04f93bcfc83a36edd7e96fe48e846f2e54fe7a7090fe8e46ba72123e1cdee0667777c38c4930e50401074d8ab31a9717457fcefaa46323003af553bee2b49ea7f907eb2ff3301463e64a8c53975c853bbdd2956b9001b5ce1562264963fce84201daaf752de6df7ca31291226969c9851d1fc4ea88ca67d38c38587c2cdd8bc4d3f7bdf705497a1e054246f684554b3b8dfac43194f1eadec7f83b711e663b5645bde6d7f8cefb59758303599fed25c3b4d2e4499d439c915910dd283b3e7118320f1c6e7385009fbcb9ae79bab72a85e644182b4dafc0a173241f2ae68ae6a504f17f102da1e91de4548c7f5bc1c107354519077a4e83407f0d6a8f0975b4ac0c2c7b30637a998dda27b56b56245371296b816876b859677bcf3473a07e0f300e788fdd60c51b1626b46050b182457c6d716994847aaef667ca45b2cede550c92d336ff29ce6effd933b875f81381cda6e59e9727e728a58c0b3e74035beeeb639ab7463744322bf40138b81895e9a8e8850c9513782dc7a79f04380c216cb177951d8940d576486b887a232fcd382adcbd639e70af0c1a08bcf1405496606fce4645aef10d769dc0c010a8a433d8cd24d5943843a89cdbc8d16531db027b312ab2c03a7f1fdb7f2bcb128639c49e86705c948137fd42d0080fda4be4e9ee812057c7974acbf0162730d3b647b355ac1a5adbb2993832eba443b7c9b5a0ae1fc00a6c0c2b0b65b9019690565739d6439bf602066a3a9bd9c67b83606de51792d25ae517cbbdf6e1827fa0e8b2b5c6023cbb1e9f0e10b786dc6fa154e282fd9c90b8d46ca685d0f4434760035073c92d131564b6845ef57457488add4f709073bbb41f5f31f8226904875a9fd9e1b7a2901e71426104d7a298a05af0d4ab549fbd69c539ebe64949a9b6088f16e2e4bc827c305cb8d64536b8364dc3d5f7519c3b431faa38b47a958cf0c6dcabf205280693abf747c262f44cd6ffa11b32fc38d4f9c3631d554d8b57389f1390ac65c06357843ee6d9f289bb054ef25de45c5149c090fe6ddcd4095696dcc9a5cfc09c8bdfd5b83a153'), packet.to_bytes()) for i, privkey in enumerate(payment_path_privkeys): - processed_packet = process_onion_packet(packet, associated_data, privkey) + processed_packet = process_onion_packet(packet, privkey, associated_data=associated_data) self.assertEqual(hops_data[i].to_bytes(), processed_packet.hop_data.to_bytes()) packet = processed_packet.next_packet @@ -398,3 +472,32 @@ class Test_LNRouter(ElectrumTestCase): self.assertEqual(4, index_of_sender) self.assertEqual(OnionFailureCode.TEMPORARY_NODE_FAILURE, failure_msg.code) self.assertEqual(b'', failure_msg.data) + + async def test_find_path_for_onion_message(self): + self.prepare_graph() + amount_to_send = 1000 # we route along channels, and we use find_path_for_payment, so dummy this. + + path = self.path_finder.find_path_for_payment( + nodeA=node('a'), + nodeB=node('c'), + invoice_amount_msat=amount_to_send, + node_filter=is_onion_message_node) + self.assertEqual([ + PathEdge(start_node=node('a'), end_node=node('d'), short_channel_id=channel(6)), + PathEdge(start_node=node('d'), end_node=node('c'), short_channel_id=channel(4)), + ], path) + + # impossible routes + path = self.path_finder.find_path_for_payment( + nodeA=node('e'), + nodeB=node('a'), + invoice_amount_msat=amount_to_send, + node_filter=is_onion_message_node) + self.assertIsNone(path) + + path = self.path_finder.find_path_for_payment( + nodeA=node('a'), + nodeB=node('e'), + invoice_amount_msat=amount_to_send, + node_filter=is_onion_message_node) + self.assertIsNone(path) diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py new file mode 100644 index 000000000..03b37fdb5 --- /dev/null +++ b/tests/test_onion_message.py @@ -0,0 +1,234 @@ +import io +import os + +import electrum_ecc as ecc + +from electrum.lnmsg import decode_msg, OnionWireSerializer +from electrum.lnonion import ( + OnionHopsDataSingle, OnionPacket, + process_onion_packet, get_bolt04_onion_key, encrypt_onionmsg_data_tlv, + get_shared_secrets_along_route, new_onion_packet, ONION_MESSAGE_LARGE_SIZE, + HOPS_DATA_SIZE, InvalidPayloadSize) +from electrum.crypto import get_ecdh +from electrum.onion_message import blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data +from electrum.util import bfh, read_json_file + +from . import ElectrumTestCase + +# test vectors https://github.com/lightning/bolts/pull/759/files +path = os.path.join(os.path.dirname(__file__), 'blinded-onion-message-onion-test.json') +test_vectors = read_json_file(path) +ONION_MESSAGE_PACKET = bfh(test_vectors['onionmessage']['onion_message_packet']) +ALICE_PUBKEY = bfh(test_vectors['route']['introduction_node_id']) +BOB_PUBKEY = bfh(test_vectors['generate']['hops'][0]['tlvs']['next_node_id']) +CAROL_PUBKEY = bfh(test_vectors['generate']['hops'][1]['tlvs']['next_node_id']) +DAVE_PUBKEY = bfh(test_vectors['generate']['hops'][2]['tlvs']['next_node_id']) + +BLINDING_SECRET = bfh(test_vectors['generate']['hops'][0]['blinding_secret']) +BLINDING_OVERRIDE_SECRET = bfh(test_vectors['generate']['hops'][0]['tlvs']['blinding_override_secret']) + +SESSION_KEY = bfh(test_vectors['generate']['session_key']) + + +class TestOnionMessage(ElectrumTestCase): + + def test_path_pubkeys_blinded_path_appended(self): + + hop_shared_secrets1, blinded_node_ids1 = get_shared_secrets_along_route([ALICE_PUBKEY], BLINDING_SECRET) + hop_shared_secrets2, blinded_node_ids2 = get_shared_secrets_along_route([BOB_PUBKEY, CAROL_PUBKEY, DAVE_PUBKEY], BLINDING_OVERRIDE_SECRET) + hop_shared_secrets = hop_shared_secrets1 + hop_shared_secrets2 + blinded_node_ids = blinded_node_ids1 + blinded_node_ids2 + + for i, ss in enumerate(hop_shared_secrets): + self.assertEqual(ss, bfh(test_vectors['generate']['hops'][i]['ss'])) + for i, ss in enumerate(blinded_node_ids): + self.assertEqual(ss, bfh(test_vectors['generate']['hops'][i]['blinded_node_id'])) + + hops_tlvs = [x['tlvs'] for x in test_vectors['generate']['hops']] + hops_data = [ + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={ + 'next_node_id': {'node_id': bfh(hops_tlvs[0]['next_node_id'])}, + 'next_blinding_override': {'blinding': bfh(hops_tlvs[0]['next_blinding_override'])}, + } + ), + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={ + 'next_node_id': {'node_id': bfh(hops_tlvs[1]['next_node_id'])}, + 'unknown_tag_561': {'data': bfh(hops_tlvs[1]['unknown_tag_561'])}, + } + ), + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={ + 'padding': {'padding': bfh(hops_tlvs[2]['padding'])}, + 'next_node_id': {'node_id': bfh(hops_tlvs[2]['next_node_id'])}, + } + ), + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + payload={'message': {'text': b'hello'}}, + blind_fields={ + 'padding': {'padding': bfh(hops_tlvs[3]['padding'])}, + 'path_id': {'data': bfh(hops_tlvs[3]['path_id'])}, + 'unknown_tag_65535': {'data': bfh(hops_tlvs[3]['unknown_tag_65535'])}, + } + ) + ] + + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + packet = new_onion_packet(blinded_node_ids, SESSION_KEY, hops_data, onion_message=True) + self.assertEqual(packet.to_bytes(), ONION_MESSAGE_PACKET) + + def test_onion_message_payload_size(self): + hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route([DAVE_PUBKEY], SESSION_KEY) + + def hops_data_for_message(message): + return [ + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + payload={'message': {'text': message.encode('utf-8')}}, + blind_fields={ + 'path_id': {'data': bfh('deadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0')}, + } + ) + ] + hops_data = hops_data_for_message('short_message') # fit in HOPS_DATA_SIZE + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + packet = new_onion_packet(blinded_node_ids, SESSION_KEY, hops_data, onion_message=True) + self.assertEqual(len(packet.to_bytes()), HOPS_DATA_SIZE + 66) + + hops_data = hops_data_for_message('A' * HOPS_DATA_SIZE) # fit in ONION_MESSAGE_LARGE_SIZE + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + packet = new_onion_packet(blinded_node_ids, SESSION_KEY, hops_data, onion_message=True) + + self.assertEqual(len(packet.to_bytes()), ONION_MESSAGE_LARGE_SIZE + 66) + + hops_data = hops_data_for_message('A' * ONION_MESSAGE_LARGE_SIZE) # does not fit in ONION_MESSAGE_LARGE_SIZE + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + with self.assertRaises(InvalidPayloadSize): + new_onion_packet(blinded_node_ids, SESSION_KEY, hops_data, onion_message=True) + + def test_decode_onion_message_packet(self): + op = OnionPacket.from_bytes(ONION_MESSAGE_PACKET) + self.assertEqual(op.hmac, bfh('8e7fc7590ff05a9e991de03f023d0aaf8688ed6170def5091c66576a424ac1cb')) + self.assertEqual(op.public_key, bfh('02531fe6068134503d2723133227c867ac8fa6c83c537e9a44c3c5bdbdcb1fe337')) + self.assertEqual(op.hops_data, bfh('93b828776d70aabbd8cef1a5b52d5a397ae1a20f20435ff6057cd8be339d5aee226660ef73b64afa45dbf2e6e8e26eb96a259b2db5aeecda1ce2e768bbc35d389d7f320ca3d2bd14e2689bef2f5ac0307eaaabc1924eb972c1563d4646ae131accd39da766257ed35ea36e4222527d1db4fa7b2000aab9eafcceed45e28b5560312d4e2299bd8d1e7fe27d10925966c28d497aec400b4630485e82efbabc00550996bdad5d6a9a8c75952f126d14ad2cff91e16198691a7ef2937de83209285f1fb90944b4e46bca7c856a9ce3da10cdf2a7d00dc2bf4f114bc4d3ed67b91cbde558ce9af86dc81fbdc37f8e301b29e23c1466659c62bdbf8cff5d4c20f0fb0851ec72f5e9385dd40fdd2e3ed67ca4517117825665e50a3e26f73c66998daf18e418e8aef9ce2d20da33c3629db2933640e03e7b44c2edf49e9b482db7b475cfd4c617ae1d46d5c24d697846f9f08561eac2b065f9b382501f6eabf07343ed6c602f61eab99cdb52adf63fd44a8db2d3016387ea708fc1c08591e19b4d9984ebe31edbd684c2ea86526dd8c7732b1d8d9117511dc1b643976d356258fce8313b1cb92682f41ab72dedd766f06de375f9edacbcd0ca8c99b865ea2b7952318ea1fd20775a28028b5cf59dece5de14f615b8df254eee63493a5111ea987224bea006d8f1b60d565eef06ac0da194dba2a6d02e79b2f2f34e9ca6e1984a507319d86e9d4fcaeea41b4b9144e0b1826304d4cc1da61cfc5f8b9850697df8adc5e9d6f3acb3219b02764b4909f2b2b22e799fd66c383414a84a7d791b899d4aa663770009eb122f90282c8cb9cda16aba6897edcf9b32951d0080c0f52be3ca011fbec3fb16423deb47744645c3b05fdbd932edf54ba6efd26e65340a8e9b1d1216582e1b30d64524f8ca2d6c5ba63a38f7120a3ed71bed8960bcac2feee2dd41c90be48e3c11ec518eb3d872779e4765a6cc28c6b0fa71ab57ced73ae963cc630edae4258cba2bf25821a6ae049fec2fca28b5dd1bb004d92924b65701b06dcf37f0ccd147a13a03f9bc0f98b7d78fe9058089756931e2cd0e0ed92ec6759d07b248069526c67e9e6ce095118fd3501ba0f858ef030b76c6f6beb11a09317b5ad25343f4b31aef02bc555951bc7791c2c289ecf94d5544dcd6ad3021ed8e8e3db34b2a73e1eedb57b578b068a5401836d6e382110b73690a94328c404af25e85a8d6b808893d1b71af6a31fadd8a8cc6e31ecc0d9ff7e6b91fd03c274a5c1f1ccd25b61150220a3fddb04c91012f5f7a83a5c90deb2470089d6e38cd5914b9c946eca6e9d31bbf8667d36cf87effc3f3ff283c21dd4137bd569fe7cf758feac94053e4baf7338bb592c8b7c291667fadf4a9bf9a2a154a18f612cbc7f851b3f8f2070e0a9d180622ee4f8e81b0ab250d504cef24116a3ff188cc829fcd8610b56343569e8dc997629410d1967ca9dd1d27eec5e01e4375aad16c46faba268524b154850d0d6fe3a76af2c6aa3e97647c51036049ac565370028d6a439a2672b6face56e1b171496c0722cfa22d9da631be359661617c5d5a2d286c5e19db9452c1e21a0107b6400debda2decb0c838f342dd017cdb2dccdf1fe97e3df3f881856b546997a3fed9e279c720145101567dd56be21688fed66bf9759e432a9aa89cbbd225d13cdea4ca05f7a45cfb6a682a3d5b1e18f7e6cf934fae5098108bae9058d05c3387a01d8d02a656d2bfff67e9f46b2d8a6aac28129e52efddf6e552214c3f8a45bc7a912cca9a7fec1d7d06412c6972cb9e3dc518983f56530b8bffe7f92c4b6eb47d4aef59fb513c4653a42de61bc17ad772')) + + def test_decode_onion_message(self): + msg = test_vectors['decrypt']['hops'][0]['onion_message'] + msgtype, data = decode_msg(bfh(msg)) + self.assertEqual(msgtype, 'onion_message') + self.assertEqual(data, { + 'blinding': bfh(test_vectors['route']['blinding']), + 'len': 1366, + 'onion_message_packet': ONION_MESSAGE_PACKET, + }) + + def test_decrypt_onion_message(self): + o = OnionPacket.from_bytes(ONION_MESSAGE_PACKET) + our_privkey = bfh(test_vectors['decrypt']['hops'][0]['privkey']) + blinding = bfh(test_vectors['route']['blinding']) + + shared_secret = get_ecdh(our_privkey, blinding) + b_hmac = get_bolt04_onion_key(b'blinded_node_id', shared_secret) + b_hmac_int = int.from_bytes(b_hmac, byteorder="big") + + our_privkey_int = int.from_bytes(our_privkey, byteorder="big") + our_privkey_int = our_privkey_int * b_hmac_int % ecc.CURVE_ORDER + our_privkey = our_privkey_int.to_bytes(32, byteorder="big") + + p = process_onion_packet(o, our_privkey, tlv_stream_name='onionmsg_tlv') + + self.assertEqual(p.hop_data.blind_fields, {}) + self.assertEqual(p.hop_data.hmac, bfh('a5296325ba478ba1e1a9d1f30a2d5052b2e2889bbd64f72c72bc71d8817288a2')) + self.assertEqual(p.hop_data.payload, {'encrypted_recipient_data': {'encrypted_recipient_data': bfh('49531cf38d3280b7f4af6d6461a2b32e3df50acfd35176fc61422a1096eed4dfc3806f29bf74320f712a61c766e7f7caac0c42f86040125fbaeec0c7613202b206dbdd31fda56394367b66a711bfd7d5bedbe20bed1b')}}) + self.assertEqual(p.hop_data.tlv_stream_name, 'onionmsg_tlv') + + onion_message_bob = test_vectors['decrypt']['hops'][1]['onion_message'] + msgtype, data = decode_msg(bfh(onion_message_bob)) + self.assertEqual(msgtype, 'onion_message') + self.assertEqual(data, { + 'blinding': bfh(test_vectors['generate']['hops'][0]['tlvs']['next_blinding_override']), + 'len': 1366, + 'onion_message_packet': p.next_packet.to_bytes(), + }) + + def test_blinding_privkey(self): + a = blinding_privkey(bfh('4141414141414141414141414141414141414141414141414141414141414141'), + bfh('031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f')) + self.assertEqual(a, bfh('7e959bf6bdd3a98caf26cbbee7b69678381d5fa2882c6c12eb2042c2367264b0')) + + def test_create_blinded_path(self): + pubkey = ALICE_PUBKEY + session_key = bfh('3030303030303030303030303030303030303030303030303030303030303030') # typo? + final_recipient_data = {'path_id': {'data': bfh('0102')}} + rp = create_blinded_path(session_key, [pubkey], final_recipient_data) + + self.assertEqual(pubkey, rp['first_node_id']) + self.assertEqual(bfh('022ed557f5ad336b31a49857e4e9664954ac33385aa20a93e2d64bfe7f08f51277'), rp['blinding']) + self.assertEqual(1, rp['num_hops']) + self.assertEqual([{ + 'blinded_node_id': bfh('031e5d91e6c417f6e8c16d1086db1887edef7be9334f5e744d04edb8da7507481e'), + 'enclen': 20, + 'encrypted_recipient_data': bfh('2dbaa54a819775aa0548ab85db68c5099e7b1180') + }], rp['path']) + + # TODO: serialization test to test_lnmsg.py + with io.BytesIO() as blinded_path_fd: + OnionWireSerializer._write_complex_field( + fd=blinded_path_fd, + field_type='blinded_path', + count=1, + value=rp) + blinded_path = blinded_path_fd.getvalue() + self.assertEqual(blinded_path, bfh('02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619022ed557f5ad336b31a49857e4e9664954ac33385aa20a93e2d64bfe7f08f5127701031e5d91e6c417f6e8c16d1086db1887edef7be9334f5e744d04edb8da7507481e00142dbaa54a819775aa0548ab85db68c5099e7b1180')) + + def prepare_blinded_path_bob_to_dave(self): + final_recipient_data = { + 'padding': {'padding': b''}, + 'path_id': {'data': bfh('deadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0')}, + 'unknown_tag_65535': {'data': bfh('06c1')} + } + hop_extras = [ + {'unknown_tag_561': {'data': bfh('123456')}}, + {'padding': {'padding': bfh('0000000000')}} + ] + return create_blinded_path(BLINDING_OVERRIDE_SECRET, [BOB_PUBKEY, CAROL_PUBKEY, DAVE_PUBKEY], final_recipient_data, hop_extras=hop_extras) + + def test_create_onionmessage_to_blinded_path_via_alice(self): + blinded_path_to_dave = self.prepare_blinded_path_bob_to_dave() + hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route([ALICE_PUBKEY], BLINDING_SECRET) + hops_data = [ + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + blind_fields={ + 'next_node_id': {'node_id': BOB_PUBKEY}, + 'next_blinding_override': {'blinding': bfh('031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f')}, + } + ), + ] + # encrypt encrypted_data_tlv here + for i in range(len(hops_data)): + encrypted_recipient_data = encrypt_onionmsg_data_tlv(shared_secret=hop_shared_secrets[i], **hops_data[i].blind_fields) + hops_data[i].payload['encrypted_recipient_data'] = {'encrypted_recipient_data': encrypted_recipient_data} + + blinded_path_blinded_ids = [] + for i, x in enumerate(blinded_path_to_dave.get('path')): + blinded_path_blinded_ids.append(x.get('blinded_node_id')) + payload = {'encrypted_recipient_data': {'encrypted_recipient_data': x.get('encrypted_recipient_data')}} + if i == len(blinded_path_to_dave.get('path')) - 1: + # add final recipient payload + payload['message'] = {'text': b'hello'} + hops_data.append( + OnionHopsDataSingle( + tlv_stream_name='onionmsg_tlv', + payload=payload) + ) + payment_path_pubkeys = blinded_node_ids + blinded_path_blinded_ids + hop_shared_secrets, _ = get_shared_secrets_along_route(payment_path_pubkeys, SESSION_KEY) + encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) + packet = new_onion_packet(payment_path_pubkeys, SESSION_KEY, hops_data, onion_message=True) + self.assertEqual(packet.to_bytes(), ONION_MESSAGE_PACKET) From 7109c22317322ac29b01061c5074437f55be43f0 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Tue, 3 Dec 2024 15:58:10 +0100 Subject: [PATCH 06/22] unasync, no add_peer in create_onion_message_route_to, add manager tests --- electrum/commands.py | 2 +- electrum/onion_message.py | 81 +++++++++++------- tests/test_onion_message.py | 159 +++++++++++++++++++++++++++++++++++- 3 files changed, 209 insertions(+), 33 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index a389dd6da..dcda79c46 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1486,7 +1486,7 @@ class Commands(Logger): } try: - await send_onion_message_to(wallet.lnworker, node_id_or_blinded_path, destination_payload) + send_onion_message_to(wallet.lnworker, node_id_or_blinded_path, destination_payload) return {'success': True} except Exception as e: msg = str(e) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 1da74a7b2..018458ba2 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: from electrum.lnworker import LNWallet from electrum.network import Network from electrum.lnrouter import NodeInfo + from electrum.lntransport import LNPeerAddr from asyncio import Task logger = get_logger(__name__) @@ -59,6 +60,12 @@ FORWARD_RETRY_DELAY = 2 FORWARD_MAX_QUEUE = 3 +class NoRouteFound(Exception): + def __init__(self, *args, peer_address: 'LNPeerAddr' = None): + Exception.__init__(self, *args) + self.peer_address = peer_address + + def create_blinded_path(session_key: bytes, path: List[bytes], final_recipient_data: dict, *, hop_extras: Optional[Sequence[dict]] = None, dummy_hops: Optional[int] = 0) -> dict: @@ -135,7 +142,7 @@ def encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets): hops_data[i].payload['encrypted_recipient_data'] = {'encrypted_recipient_data': encrypted_recipient_data} -async def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> List[PathEdge]: +def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> List[PathEdge]: """Constructs a route to the destination node_id, first by starting with peers with existing channels, and if no route found, opening a direct peer connection if node_id is found with an address in channel_db.""" @@ -145,7 +152,7 @@ async def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> chan.is_active() and not chan.is_frozen_for_sending()] my_sending_channels = {chan.short_channel_id: chan for chan in my_active_channels if chan.short_channel_id is not None} - # strat1: find route to introduction point over existing channel mesh + # find route to introduction point over existing channel mesh # NOTE: nodes that are in channel_db but are offline are not removed from the set if lnwallet.network.path_finder: if path := lnwallet.network.path_finder.find_path_for_payment( @@ -156,17 +163,19 @@ async def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> my_sending_channels=my_sending_channels ): return path - # strat2: dest node has host:port in channel_db? then open direct peer connection + # alt: dest is existing peer? + if lnwallet.peers.get(node_id): + return [PathEdge(short_channel_id=None, start_node=None, end_node=node_id)] + + # if we have an address, pass it. if lnwallet.channel_db: if peer_addr := lnwallet.channel_db.get_last_good_address(node_id): - peer = await lnwallet.add_peer(str(peer_addr)) - await peer.initialized - return [PathEdge(short_channel_id=None, start_node=None, end_node=node_id)] + raise NoRouteFound('no path found, peer_addr available', peer_address=peer_addr) - raise Exception('no path found') + raise NoRouteFound('no path found') -async def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: bytes, destination_payload: dict, session_key: bytes = None): +def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: bytes, destination_payload: dict, session_key: bytes = None): if session_key is None: session_key = os.urandom(32) @@ -226,7 +235,7 @@ async def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: b # start of blinded path is our peer blinding = blinded_path['blinding'] else: - path = await create_onion_message_route_to(lnwallet, introduction_point) + path = create_onion_message_route_to(lnwallet, introduction_point) # first edge must be to our peer peer = lnwallet.peers.get(path[0].end_node) @@ -303,7 +312,7 @@ async def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: b # destination is our direct peer, no need to route-find path = [PathEdge(short_channel_id=None, start_node=None, end_node=pubkey)] else: - path = await create_onion_message_route_to(lnwallet, pubkey) + path = create_onion_message_route_to(lnwallet, pubkey) # first edge must be to our peer peer = lnwallet.peers.get(path[0].end_node) @@ -340,9 +349,9 @@ async def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: b ) -async def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, - max_paths: int = REQUEST_REPLY_PATHS_MAX, - preferred_node_id: bytes = None) -> List[dict]: +def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, + max_paths: int = REQUEST_REPLY_PATHS_MAX, + preferred_node_id: bytes = None) -> List[dict]: # TODO: build longer paths and/or add dummy hops to increase privacy my_active_channels = [chan for chan in lnwallet.channels.values() if chan.is_active()] my_onionmsg_channels = [chan for chan in my_active_channels if lnwallet.peers.get(chan.node_id) and @@ -376,7 +385,9 @@ class OnionMessageManager(Logger): - association between onion message and their replies - manage re-send attempts, TODO: iterate through routes (both directions)""" - def __init__(self, lnwallet: 'LNWallet'): + def __init__(self, lnwallet: 'LNWallet', *, + request_reply_timeout=REQUEST_REPLY_TIMEOUT, + request_reply_retry_delay=REQUEST_REPLY_RETRY_DELAY): Logger.__init__(self) self.network = None # type: Optional['Network'] self.taskgroup = None # type: OldTaskGroup @@ -389,6 +400,9 @@ class OnionMessageManager(Logger): self.forwardqueue = queue.PriorityQueue() self.forwardqueue_notempty = asyncio.Event() + self.request_reply_timeout = request_reply_timeout + self.request_reply_retry_delay = request_reply_retry_delay + def start_network(self, *, network: 'Network'): assert network assert self.network is None, "already started" @@ -415,13 +429,13 @@ class OnionMessageManager(Logger): try: scheduled, expires, onion_packet, blinding, node_id = self.forwardqueue.get_nowait() except queue.Empty: - self.logger.debug(f'fwd queue empty') + self.logger.info(f'forward queue empty') self.forwardqueue_notempty.clear() await self.forwardqueue_notempty.wait() continue if expires <= now(): - self.logger.debug(f'fwd expired {node_id=}') + self.logger.debug(f'forward expired {node_id=}') continue if scheduled > now(): # return to queue @@ -448,7 +462,7 @@ class OnionMessageManager(Logger): blinding: bytes, node_id: bytes): if self.forwardqueue.qsize() >= FORWARD_MAX_QUEUE: - self.logger.debug('fwd queue full, dropping packet') + self.logger.debug('forward queue full, dropping packet') return expires = now() + FORWARD_RETRY_TIMEOUT queueitem = (now(), expires, onion_packet, blinding, node_id) @@ -460,9 +474,13 @@ class OnionMessageManager(Logger): try: scheduled, expires, key = self.requestreply_queue.get_nowait() except queue.Empty: - self.logger.debug(f'requestreply queue empty') + self.logger.info(f'requestreply queue empty') self.requestreply_queue_notempty.clear() - await self.requestreply_queue_notempty.wait() + try: + self.requestreply_queue_notempty.clear() + await self.requestreply_queue_notempty.wait() # NOTE: quirk, see note below + except Exception as e: + self.logger.info(f'Exception e={e!r}') continue requestreply = self.get_requestreply(key) @@ -483,12 +501,17 @@ class OnionMessageManager(Logger): continue try: - await self._send_pending_requestreply(key) + self._send_pending_requestreply(key) except BaseException as e: - self.logger.debug(f'error while sending {key=}') - self._set_requestreply_result(key, e) + self.logger.debug(f'error while sending {key=} {e!r}') + self._set_requestreply_result(key, copy.copy(e)) + # NOTE: above, when passing the caught exception instance e directly it leads to GeneratorExit() in + # queue_notempty.wait() later (??). pass a copy instead. + if isinstance(e, NoRouteFound) and e.peer_address: + await self.lnwallet.add_peer(str(e.peer_address)) else: - self.requestreply_queue.put_nowait((now() + REQUEST_REPLY_RETRY_DELAY, expires, key)) + self.logger.debug(f'resubmit {key=}') + self.requestreply_queue.put_nowait((now() + self.request_reply_retry_delay, expires, key)) def get_requestreply(self, key): with self.pending_lock: @@ -498,6 +521,7 @@ class OnionMessageManager(Logger): with self.pending_lock: requestreply = self.pending.get(key) if requestreply is None: + self.logger.error(f'requestreply with {key=} not found!') return self.pending[key]['result'] = result requestreply['ev'].set() @@ -537,7 +561,7 @@ class OnionMessageManager(Logger): } # tuple = (when to process, when it expires, key) - expires = now() + REQUEST_REPLY_TIMEOUT + expires = now() + self.request_reply_timeout queueitem = (now(), expires, key) self.requestreply_queue.put_nowait(queueitem) task = asyncio.create_task(self._requestreply_task(key)) @@ -560,12 +584,12 @@ class OnionMessageManager(Logger): assert requestreply result = requestreply.get('result') if isinstance(result, Exception): - raise result + raise result # raising in the task requires caller to explicitly extract exception. return result finally: self._remove_requestreply(key) - async def _send_pending_requestreply(self, key): + def _send_pending_requestreply(self, key): """adds reply_path to payload""" data = self.get_requestreply(key) payload = data.get('payload') @@ -577,7 +601,7 @@ class OnionMessageManager(Logger): if 'reply_path' not in final_payload: # unless explicitly set in payload, generate reply_path here path_id = self._path_id_from_payload_and_key(payload, key) - reply_paths = await get_blinded_reply_paths(self.lnwallet, path_id, max_paths=1) + reply_paths = get_blinded_reply_paths(self.lnwallet, path_id, max_paths=1) if not reply_paths: raise Exception(f'Could not create a reply_path for {key=}') @@ -585,10 +609,9 @@ class OnionMessageManager(Logger): # TODO: we should try alternate paths when retrying, this is currently not done. # (send_onion_message_to decides path, without knowledge of prev attempts) - await send_onion_message_to(self.lnwallet, node_id_or_blinded_path, final_payload) + send_onion_message_to(self.lnwallet, node_id_or_blinded_path, final_payload) def _path_id_from_payload_and_key(self, payload: dict, key: bytes) -> bytes: - # TODO: construct path_id in such a way that we can determine the request originated from us and is not spoofed # TODO: use payload to determine prefix? return b'electrum' + key diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 03b37fdb5..0d58fe543 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -1,7 +1,11 @@ +import asyncio import io import os +import time +from functools import partial import electrum_ecc as ecc +from electrum_ecc import ECPrivkey from electrum.lnmsg import decode_msg, OnionWireSerializer from electrum.lnonion import ( @@ -10,10 +14,13 @@ from electrum.lnonion import ( get_shared_secrets_along_route, new_onion_packet, ONION_MESSAGE_LARGE_SIZE, HOPS_DATA_SIZE, InvalidPayloadSize) from electrum.crypto import get_ecdh -from electrum.onion_message import blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data -from electrum.util import bfh, read_json_file +from electrum.lnutil import LnFeatures +from electrum.onion_message import blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data, \ + OnionMessageManager, NoRouteFound, Timeout +from electrum.util import bfh, read_json_file, OldTaskGroup, get_asyncio_loop -from . import ElectrumTestCase +from . import ElectrumTestCase, test_lnpeer +from .test_lnpeer import PutIntoOthersQueueTransport, PeerInTests, keypair # test vectors https://github.com/lightning/bolts/pull/759/files path = os.path.join(os.path.dirname(__file__), 'blinded-onion-message-onion-test.json') @@ -232,3 +239,149 @@ class TestOnionMessage(ElectrumTestCase): encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets) packet = new_onion_packet(payment_path_pubkeys, SESSION_KEY, hops_data, onion_message=True) self.assertEqual(packet.to_bytes(), ONION_MESSAGE_PACKET) + + +class MockNetwork: + def __init__(self): + self.asyncio_loop = get_asyncio_loop() + self.taskgroup = OldTaskGroup() + + +class MockWallet: + def __init__(self): + pass + + +class MockLNWallet(test_lnpeer.MockLNWallet): + + async def add_peer(self, connect_str: str): + t1 = PutIntoOthersQueueTransport(self.node_keypair, 'test') + p1 = PeerInTests(self, keypair().pubkey, t1) + self.peers[p1.pubkey] = p1 + p1.initialized.set_result(True) + return p1 + + +class MockPeer: + their_features = LnFeatures(LnFeatures.OPTION_ONION_MESSAGE_OPT) + + def __init__(self, pubkey, on_send_message=None): + self.pubkey = pubkey + self.on_send_message = on_send_message + + async def wait_one_htlc_switch_iteration(self, *args): + pass + + def send_message(self, *args, **kwargs): + if self.on_send_message: + self.on_send_message(*args, **kwargs) + + +class TestOnionMessageManager(ElectrumTestCase): + + def setUp(self): + super().setUp() + self.alice = ECPrivkey(privkey_bytes=b'\x41'*32) + self.alice_pub = self.alice.get_public_key_bytes(compressed=True) + self.bob = ECPrivkey(privkey_bytes=b'\x42'*32) + self.bob_pub = self.bob.get_public_key_bytes(compressed=True) + self.carol = ECPrivkey(privkey_bytes=b'\x43'*32) + self.carol_pub = self.carol.get_public_key_bytes(compressed=True) + self.dave = ECPrivkey(privkey_bytes=b'\x44'*32) + self.dave_pub = self.dave.get_public_key_bytes(compressed=True) + self.eve = ECPrivkey(privkey_bytes=b'\x45'*32) + self.eve_pub = self.eve.get_public_key_bytes(compressed=True) + + async def run_test1(self, t): + t1 = t.submit_requestreply( + payload={'message': {'text': 'alice_timeout'.encode('utf-8')}}, + node_id_or_blinded_path=self.alice_pub) + + with self.assertRaises(Timeout): + await t1 + + async def run_test2(self, t): + t2 = t.submit_requestreply( + payload={'message': {'text': 'bob_slow_timeout'.encode('utf-8')}}, + node_id_or_blinded_path=self.bob_pub) + + with self.assertRaises(Timeout): + await t2 + + async def run_test3(self, t, rkey): + t3 = t.submit_requestreply( + payload={'message': {'text': 'carol_with_immediate_reply'.encode('utf-8')}}, + node_id_or_blinded_path=self.carol_pub, + key=rkey) + + t3_result = await t3 + self.assertEqual(t3_result, ({'path_id': {'data': b'electrum' + rkey}}, {})) + + async def run_test4(self, t, rkey): + t4 = t.submit_requestreply( + payload={'message': {'text': 'dave_with_slow_reply'.encode('utf-8')}}, + node_id_or_blinded_path=self.dave_pub, + key=rkey) + + t4_result = await t4 + self.assertEqual(t4_result, ({'path_id': {'data': b'electrum' + rkey}}, {})) + + async def run_test5(self, t): + t5 = t.submit_requestreply( + payload={'message': {'text': 'no_peer'.encode('utf-8')}}, + node_id_or_blinded_path=self.eve_pub) + + with self.assertRaises(NoRouteFound): + await t5 + + async def test_manager(self): + n = MockNetwork() + k = keypair() + q1, q2 = asyncio.Queue(), asyncio.Queue() + lnw = MockLNWallet(local_keypair=k, chans=[], tx_queue=q1, name='test', has_anchors=False) + + def slow(*args, **kwargs): + time.sleep(2) + + def withreply(key, *args, **kwargs): + t.on_onion_message_received_reply({'path_id': {'data': b'electrum' + key}}, {}) + + def slowwithreply(key, *args, **kwargs): + time.sleep(2) + t.on_onion_message_received_reply({'path_id': {'data': b'electrum' + key}}, {}) + + rkey1 = bfh('0102030405060708') + rkey2 = bfh('0102030405060709') + + lnw.peers[self.alice_pub] = MockPeer(self.alice_pub) + lnw.peers[self.bob_pub] = MockPeer(self.bob_pub, on_send_message=slow) + lnw.peers[self.carol_pub] = MockPeer(self.carol_pub, on_send_message=partial(withreply, rkey1)) + lnw.peers[self.dave_pub] = MockPeer(self.dave_pub, on_send_message=partial(slowwithreply, rkey2)) + t = OnionMessageManager(lnw, request_reply_timeout=5, request_reply_retry_delay=1) + t.start_network(network=n) + + try: + await asyncio.sleep(1) + + self.logger.debug('tests in sequence') + + await self.run_test1(t) + await self.run_test2(t) + await self.run_test3(t, rkey1) + await self.run_test4(t, rkey2) + await self.run_test5(t) + + self.logger.debug('tests in parallel') + + async with OldTaskGroup() as group: + await group.spawn(self.run_test1(t)) + await group.spawn(self.run_test2(t)) + await group.spawn(self.run_test3(t, rkey1)) + await group.spawn(self.run_test4(t, rkey2)) + await group.spawn(self.run_test5(t)) + finally: + await asyncio.sleep(1) + + self.logger.debug('stopping manager') + await t.stop() + await lnw.stop() From a157108e75227f1b8ff9d0a455f02bdf1961d3f9 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Wed, 22 Jan 2025 16:43:33 +0100 Subject: [PATCH 07/22] onion_messages: fix code indentation --- electrum/commands.py | 9 +++-- electrum/lnmsg.py | 85 ++++++++++++++++++++++----------------- electrum/onion_message.py | 31 +++++++------- 3 files changed, 70 insertions(+), 55 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index dcda79c46..6e51fdcdd 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1516,10 +1516,11 @@ class Commands(Logger): blinded_path = create_blinded_path(session_key, path=path, final_recipient_data={}, dummy_hops=dummy_hops) with io.BytesIO() as blinded_path_fd: - OnionWireSerializer._write_complex_field(fd=blinded_path_fd, - field_type='blinded_path', - count=1, - value=blinded_path) + OnionWireSerializer._write_complex_field( + fd=blinded_path_fd, + field_type='blinded_path', + count=1, + value=blinded_path) encoded_blinded_path = blinded_path_fd.getvalue() return encoded_blinded_path.hex() diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index ec217dc2d..da849f22f 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -420,23 +420,26 @@ class LNSerializer: subtype_field_type = row[3] subtype_field_count_str = row[4] - subtype_field_count = _resolve_field_count(subtype_field_count_str, - vars_dict=record, - allow_any=True) + subtype_field_count = _resolve_field_count( + subtype_field_count_str, + vars_dict=record, + allow_any=True) if subtype_field_name not in record: raise Exception(f'complex field type {field_type} missing element {subtype_field_name}') if subtype_field_type in self.subtypes: - self._write_complex_field(fd=fd, - field_type=subtype_field_type, - count=subtype_field_count, - value=record[subtype_field_name]) + self._write_complex_field( + fd=fd, + field_type=subtype_field_type, + count=subtype_field_count, + value=record[subtype_field_name]) else: - _write_field(fd=fd, - field_type=subtype_field_type, - count=subtype_field_count, - value=record[subtype_field_name]) + _write_field( + fd=fd, + field_type=subtype_field_type, + count=subtype_field_count, + value=record[subtype_field_name]) def _read_complex_field(self, *, fd: io.BytesIO, field_type: str, count: Union[int, str])\ -> Union[bytes, List[Dict[str, Any]], Dict[str, Any]]: @@ -460,18 +463,21 @@ class LNSerializer: subtype_field_type = row[3] subtype_field_count_str = row[4] - subtype_field_count = _resolve_field_count(subtype_field_count_str, - vars_dict=parsed, - allow_any=True) + subtype_field_count = _resolve_field_count( + subtype_field_count_str, + vars_dict=parsed, + allow_any=True) if subtype_field_type in self.subtypes: - parsed[subtype_field_name] = self._read_complex_field(fd=fd, - field_type=subtype_field_type, - count=subtype_field_count) + parsed[subtype_field_name] = self._read_complex_field( + fd=fd, + field_type=subtype_field_type, + count=subtype_field_count) else: - parsed[subtype_field_name] = _read_field(fd=fd, - field_type=subtype_field_type, - count=subtype_field_count) + parsed[subtype_field_name] = _read_field( + fd=fd, + field_type=subtype_field_type, + count=subtype_field_count) parsedlist.append(parsed) return parsedlist if count == '...' or count > 1 else parsedlist[0] @@ -498,15 +504,17 @@ class LNSerializer: allow_any=True) field_value = kwargs[tlv_record_name][field_name] if field_type in self.subtypes: - self._write_complex_field(fd=tlv_record_fd, - field_type=field_type, - count=field_count, - value=field_value) + self._write_complex_field( + fd=tlv_record_fd, + field_type=field_type, + count=field_count, + value=field_value) else: - _write_field(fd=tlv_record_fd, - field_type=field_type, - count=field_count, - value=field_value) + _write_field( + fd=tlv_record_fd, + field_type=field_type, + count=field_count, + value=field_value) else: raise Exception(f"unexpected row in scheme: {row!r}") _write_tlv_record(fd=fd, tlv_type=tlv_record_type, tlv_val=tlv_record_fd.getvalue()) @@ -544,18 +552,21 @@ class LNSerializer: field_name = row[3] field_type = row[4] field_count_str = row[5] - field_count = _resolve_field_count(field_count_str, - vars_dict=parsed[tlv_record_name], - allow_any=True) + field_count = _resolve_field_count( + field_count_str, + vars_dict=parsed[tlv_record_name], + allow_any=True) #print(f">> count={field_count}. parsed={parsed}") if field_type in self.subtypes: - parsed[tlv_record_name][field_name] = self._read_complex_field(fd=tlv_record_fd, - field_type=field_type, - count=field_count) + parsed[tlv_record_name][field_name] = self._read_complex_field( + fd=tlv_record_fd, + field_type=field_type, + count=field_count) else: - parsed[tlv_record_name][field_name] = _read_field(fd=tlv_record_fd, - field_type=field_type, - count=field_count) + parsed[tlv_record_name][field_name] = _read_field( + fd=tlv_record_fd, + field_type=field_type, + count=field_count) else: raise Exception(f"unexpected row in scheme: {row!r}") if _num_remaining_bytes_to_read(tlv_record_fd) > 0: diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 018458ba2..77ddfd094 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -264,17 +264,18 @@ def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: bytes, # final hop pre-ip, add next_blinding_override final_hop_pre_ip = OnionHopsDataSingle( tlv_stream_name='onionmsg_tlv', - blind_fields={'next_node_id': {'node_id': introduction_point}, - 'next_blinding_override': {'blinding': blinded_path['blinding']}, - } + blind_fields={ + 'next_node_id': {'node_id': introduction_point}, + 'next_blinding_override': {'blinding': blinded_path['blinding']}, + } ) - hops_data.append(final_hop_pre_ip) # encrypt encrypted_data_tlv here for i in range(len(hops_data)): - encrypted_recipient_data = encrypt_onionmsg_data_tlv(shared_secret=hop_shared_secrets[i], - **hops_data[i].blind_fields) + encrypted_recipient_data = encrypt_onionmsg_data_tlv( + shared_secret=hop_shared_secrets[i], + **hops_data[i].blind_fields) hops_data[i].payload['encrypted_recipient_data'] = { 'encrypted_recipient_data': encrypted_recipient_data } @@ -457,10 +458,11 @@ class OnionMessageManager(Logger): self.logger.debug(f'error while sending {node_id=} e={e!r}') self.forwardqueue.put_nowait((now() + FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) - def submit_forward(self, *, - onion_packet: OnionPacket, - blinding: bytes, - node_id: bytes): + def submit_forward( + self, *, + onion_packet: OnionPacket, + blinding: bytes, + node_id: bytes): if self.forwardqueue.qsize() >= FORWARD_MAX_QUEUE: self.logger.debug('forward queue full, dropping packet') return @@ -534,10 +536,11 @@ class OnionMessageManager(Logger): requestreply['ev'].set() del self.pending[key] - def submit_requestreply(self, *, - payload: dict, - node_id_or_blinded_path: bytes, - key: bytes = None) -> 'Task': + def submit_requestreply( + self, *, + payload: dict, + node_id_or_blinded_path: bytes, + key: bytes = None) -> 'Task': """Add onion message to queue for sending. Queued onion message payloads are supplied with a path_id and a reply_path to determine which request corresponds with arriving replies. From 4efd4a0ff7bb25bd6e9f9cb9505ef1dbbe8c083f Mon Sep 17 00:00:00 2001 From: ThomasV Date: Wed, 22 Jan 2025 15:09:40 +0100 Subject: [PATCH 08/22] test_onion_message: enable logging so that we can see what is going on --- tests/test_onion_message.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 0d58fe543..04ddfe949 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -3,6 +3,7 @@ import io import os import time from functools import partial +import logging import electrum_ecc as ecc from electrum_ecc import ECPrivkey @@ -18,6 +19,7 @@ from electrum.lnutil import LnFeatures from electrum.onion_message import blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data, \ OnionMessageManager, NoRouteFound, Timeout from electrum.util import bfh, read_json_file, OldTaskGroup, get_asyncio_loop +from electrum.logging import console_stderr_handler from . import ElectrumTestCase, test_lnpeer from .test_lnpeer import PutIntoOthersQueueTransport, PeerInTests, keypair @@ -279,6 +281,11 @@ class MockPeer: class TestOnionMessageManager(ElectrumTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + console_stderr_handler.setLevel(logging.DEBUG) + def setUp(self): super().setUp() self.alice = ECPrivkey(privkey_bytes=b'\x41'*32) From 3314ee1de12a4237d090b7edf5ccce76e9d8cfcc Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 16 Jan 2025 17:09:45 +0100 Subject: [PATCH 09/22] test_onion_messages: cleanup time constants, make speedup homogeneous. 1. Do not pass request_reply_timeout and request_reply_retry_delay to OnionMessageManager. Arguments passed to a function are useful only if their value might change during the execution of a program. If that is not the case, it is preferable to define them as constants. That makes the code easier to read, because the reader can focus on arguments that are actually relevant. 2 . Multiply all time constants by the same factor, instead of doing so incompletely and on a per parameter basis. (note that before this commit, the speedup factor was not consistent) 3. Do not use util.now(), because it is rounded as integer. With these changes, the tests can effectively run with a 100x speedup --- electrum/onion_message.py | 42 ++++++++++++++++++++----------------- tests/test_onion_message.py | 21 +++++++++++-------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 77ddfd094..beb020a31 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -26,6 +26,7 @@ import io import os import queue import threading +import time from random import random from typing import TYPE_CHECKING, Optional, List, Sequence @@ -40,7 +41,12 @@ from electrum.lnonion import (get_bolt04_onion_key, OnionPacket, process_onion_p OnionHopsDataSingle, decrypt_onionmsg_data_tlv, encrypt_onionmsg_data_tlv, get_shared_secrets_along_route, new_onion_packet) from electrum.lnutil import LnFeatures -from electrum.util import OldTaskGroup, now +from electrum.util import OldTaskGroup + +# do not use util.now, because it rounds to integers +def now(): + return time.time() + if TYPE_CHECKING: from electrum.lnworker import LNWallet @@ -52,12 +58,7 @@ if TYPE_CHECKING: logger = get_logger(__name__) -REQUEST_REPLY_TIMEOUT = 30 -REQUEST_REPLY_RETRY_DELAY = 5 REQUEST_REPLY_PATHS_MAX = 3 -FORWARD_RETRY_TIMEOUT = 4 -FORWARD_RETRY_DELAY = 2 -FORWARD_MAX_QUEUE = 3 class NoRouteFound(Exception): @@ -386,9 +387,14 @@ class OnionMessageManager(Logger): - association between onion message and their replies - manage re-send attempts, TODO: iterate through routes (both directions)""" - def __init__(self, lnwallet: 'LNWallet', *, - request_reply_timeout=REQUEST_REPLY_TIMEOUT, - request_reply_retry_delay=REQUEST_REPLY_RETRY_DELAY): + SLEEP_DELAY = 1 + REQUEST_REPLY_TIMEOUT = 30 + REQUEST_REPLY_RETRY_DELAY = 5 + FORWARD_RETRY_TIMEOUT = 4 + FORWARD_RETRY_DELAY = 2 + FORWARD_MAX_QUEUE = 3 + + def __init__(self, lnwallet: 'LNWallet'): Logger.__init__(self) self.network = None # type: Optional['Network'] self.taskgroup = None # type: OldTaskGroup @@ -401,9 +407,6 @@ class OnionMessageManager(Logger): self.forwardqueue = queue.PriorityQueue() self.forwardqueue_notempty = asyncio.Event() - self.request_reply_timeout = request_reply_timeout - self.request_reply_retry_delay = request_reply_retry_delay - def start_network(self, *, network: 'Network'): assert network assert self.network is None, "already started" @@ -441,7 +444,7 @@ class OnionMessageManager(Logger): if scheduled > now(): # return to queue self.forwardqueue.put_nowait((scheduled, expires, onion_packet, blinding, node_id)) - await asyncio.sleep(1) # sleep here, as the first queue item wasn't due yet + await asyncio.sleep(self.SLEEP_DELAY) # sleep here, as the first queue item wasn't due yet continue try: @@ -456,17 +459,17 @@ class OnionMessageManager(Logger): ) except BaseException as e: self.logger.debug(f'error while sending {node_id=} e={e!r}') - self.forwardqueue.put_nowait((now() + FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) + self.forwardqueue.put_nowait((now() + self.FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) def submit_forward( self, *, onion_packet: OnionPacket, blinding: bytes, node_id: bytes): - if self.forwardqueue.qsize() >= FORWARD_MAX_QUEUE: + if self.forwardqueue.qsize() >= self.FORWARD_MAX_QUEUE: self.logger.debug('forward queue full, dropping packet') return - expires = now() + FORWARD_RETRY_TIMEOUT + expires = now() + self.FORWARD_RETRY_TIMEOUT queueitem = (now(), expires, onion_packet, blinding, node_id) self.forwardqueue.put_nowait(queueitem) self.forwardqueue_notempty.set() @@ -498,8 +501,9 @@ class OnionMessageManager(Logger): continue if scheduled > now(): # return to queue + self.logger.debug(f'return to queue {key=}, {scheduled - now()}') self.requestreply_queue.put_nowait((scheduled, expires, key)) - await asyncio.sleep(1) # sleep here, as the first queue item wasn't due yet + await asyncio.sleep(self.SLEEP_DELAY) # sleep here, as the first queue item wasn't due yet continue try: @@ -513,7 +517,7 @@ class OnionMessageManager(Logger): await self.lnwallet.add_peer(str(e.peer_address)) else: self.logger.debug(f'resubmit {key=}') - self.requestreply_queue.put_nowait((now() + self.request_reply_retry_delay, expires, key)) + self.requestreply_queue.put_nowait((now() + self.REQUEST_REPLY_RETRY_DELAY, expires, key)) def get_requestreply(self, key): with self.pending_lock: @@ -564,7 +568,7 @@ class OnionMessageManager(Logger): } # tuple = (when to process, when it expires, key) - expires = now() + self.request_reply_timeout + expires = now() + self.REQUEST_REPLY_TIMEOUT queueitem = (now(), expires, key) self.requestreply_queue.put_nowait(queueitem) task = asyncio.create_task(self._requestreply_task(key)) diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 04ddfe949..46bdc609a 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -24,6 +24,13 @@ from electrum.logging import console_stderr_handler from . import ElectrumTestCase, test_lnpeer from .test_lnpeer import PutIntoOthersQueueTransport, PeerInTests, keypair +TIME_STEP = 0.01 # run tests 100 x faster +OnionMessageManager.SLEEP_DELAY *= TIME_STEP +OnionMessageManager.REQUEST_REPLY_TIMEOUT *= TIME_STEP +OnionMessageManager.REQUEST_REPLY_RETRY_DELAY *= TIME_STEP +OnionMessageManager.FORWARD_RETRY_TIMEOUT *= TIME_STEP +OnionMessageManager.FORWARD_RETRY_DELAY *= TIME_STEP + # test vectors https://github.com/lightning/bolts/pull/759/files path = os.path.join(os.path.dirname(__file__), 'blinded-onion-message-onion-test.json') test_vectors = read_json_file(path) @@ -348,13 +355,13 @@ class TestOnionMessageManager(ElectrumTestCase): lnw = MockLNWallet(local_keypair=k, chans=[], tx_queue=q1, name='test', has_anchors=False) def slow(*args, **kwargs): - time.sleep(2) + time.sleep(2*TIME_STEP) def withreply(key, *args, **kwargs): t.on_onion_message_received_reply({'path_id': {'data': b'electrum' + key}}, {}) def slowwithreply(key, *args, **kwargs): - time.sleep(2) + time.sleep(2*TIME_STEP) t.on_onion_message_received_reply({'path_id': {'data': b'electrum' + key}}, {}) rkey1 = bfh('0102030405060708') @@ -364,22 +371,18 @@ class TestOnionMessageManager(ElectrumTestCase): lnw.peers[self.bob_pub] = MockPeer(self.bob_pub, on_send_message=slow) lnw.peers[self.carol_pub] = MockPeer(self.carol_pub, on_send_message=partial(withreply, rkey1)) lnw.peers[self.dave_pub] = MockPeer(self.dave_pub, on_send_message=partial(slowwithreply, rkey2)) - t = OnionMessageManager(lnw, request_reply_timeout=5, request_reply_retry_delay=1) + t = OnionMessageManager(lnw) t.start_network(network=n) try: - await asyncio.sleep(1) - + await asyncio.sleep(TIME_STEP) self.logger.debug('tests in sequence') - await self.run_test1(t) await self.run_test2(t) await self.run_test3(t, rkey1) await self.run_test4(t, rkey2) await self.run_test5(t) - self.logger.debug('tests in parallel') - async with OldTaskGroup() as group: await group.spawn(self.run_test1(t)) await group.spawn(self.run_test2(t)) @@ -387,7 +390,7 @@ class TestOnionMessageManager(ElectrumTestCase): await group.spawn(self.run_test4(t, rkey2)) await group.spawn(self.run_test5(t)) finally: - await asyncio.sleep(1) + await asyncio.sleep(TIME_STEP) self.logger.debug('stopping manager') await t.stop() From 86432f55ee48039b65285d47e10bd60b1376ef51 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 16 Jan 2025 18:33:33 +0100 Subject: [PATCH 10/22] Remove redundant code: this line is duplicated 2 lines below --- electrum/onion_message.py | 1 - 1 file changed, 1 deletion(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index beb020a31..b01fd90d2 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -480,7 +480,6 @@ class OnionMessageManager(Logger): scheduled, expires, key = self.requestreply_queue.get_nowait() except queue.Empty: self.logger.info(f'requestreply queue empty') - self.requestreply_queue_notempty.clear() try: self.requestreply_queue_notempty.clear() await self.requestreply_queue_notempty.wait() # NOTE: quirk, see note below From aeaec452a07e1f8020c85f90677dcf215344ba02 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 16 Jan 2025 18:43:36 +0100 Subject: [PATCH 11/22] Remove unnecessary events We do not need asyncio Events in order to signal that a queue is empty. Instead, we should use asyncio queues. --- electrum/onion_message.py | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index b01fd90d2..f043fca58 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -24,7 +24,6 @@ import asyncio import copy import io import os -import queue import threading import time from random import random @@ -402,10 +401,8 @@ class OnionMessageManager(Logger): self.pending = {} self.pending_lock = threading.Lock() - self.requestreply_queue = queue.PriorityQueue() - self.requestreply_queue_notempty = asyncio.Event() - self.forwardqueue = queue.PriorityQueue() - self.forwardqueue_notempty = asyncio.Event() + self.requestreply_queue = asyncio.PriorityQueue() + self.forwardqueue = asyncio.PriorityQueue() def start_network(self, *, network: 'Network'): assert network @@ -430,14 +427,7 @@ class OnionMessageManager(Logger): async def process_forward_queue(self): while True: - try: - scheduled, expires, onion_packet, blinding, node_id = self.forwardqueue.get_nowait() - except queue.Empty: - self.logger.info(f'forward queue empty') - self.forwardqueue_notempty.clear() - await self.forwardqueue_notempty.wait() - continue - + scheduled, expires, onion_packet, blinding, node_id = await self.forwardqueue.get() if expires <= now(): self.logger.debug(f'forward expired {node_id=}') continue @@ -472,21 +462,10 @@ class OnionMessageManager(Logger): expires = now() + self.FORWARD_RETRY_TIMEOUT queueitem = (now(), expires, onion_packet, blinding, node_id) self.forwardqueue.put_nowait(queueitem) - self.forwardqueue_notempty.set() async def process_request_reply_queue(self): while True: - try: - scheduled, expires, key = self.requestreply_queue.get_nowait() - except queue.Empty: - self.logger.info(f'requestreply queue empty') - try: - self.requestreply_queue_notempty.clear() - await self.requestreply_queue_notempty.wait() # NOTE: quirk, see note below - except Exception as e: - self.logger.info(f'Exception e={e!r}') - continue - + scheduled, expires, key = await self.requestreply_queue.get() requestreply = self.get_requestreply(key) if requestreply is None: self.logger.debug(f'no data for key {key=}') @@ -511,7 +490,6 @@ class OnionMessageManager(Logger): self.logger.debug(f'error while sending {key=} {e!r}') self._set_requestreply_result(key, copy.copy(e)) # NOTE: above, when passing the caught exception instance e directly it leads to GeneratorExit() in - # queue_notempty.wait() later (??). pass a copy instead. if isinstance(e, NoRouteFound) and e.peer_address: await self.lnwallet.add_peer(str(e.peer_address)) else: @@ -571,7 +549,6 @@ class OnionMessageManager(Logger): queueitem = (now(), expires, key) self.requestreply_queue.put_nowait(queueitem) task = asyncio.create_task(self._requestreply_task(key)) - self.requestreply_queue_notempty.set() return task async def _requestreply_task(self, key): From d8147964848d188b4b0090e2658c53da8e126480 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Fri, 17 Jan 2025 10:19:08 +0100 Subject: [PATCH 12/22] Deobfuscate names The name 'requestreply' do not mean anything. If something can either be a request or a reply, perhaps we can call it 'message', instead of introducing a new name? In general, coming up with new names comes at a cost, because you are forcing other developers to learn and use your terminology. Please minimize that. --- electrum/onion_message.py | 64 ++++++++++++++++++------------------- tests/test_onion_message.py | 10 +++--- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index f043fca58..64f89927f 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -401,8 +401,8 @@ class OnionMessageManager(Logger): self.pending = {} self.pending_lock = threading.Lock() - self.requestreply_queue = asyncio.PriorityQueue() - self.forwardqueue = asyncio.PriorityQueue() + self.send_queue = asyncio.PriorityQueue() + self.forward_queue = asyncio.PriorityQueue() def start_network(self, *, network: 'Network'): assert network @@ -415,7 +415,7 @@ class OnionMessageManager(Logger): self.logger.info("starting taskgroup.") try: async with self.taskgroup as group: - await group.spawn(self.process_request_reply_queue()) + await group.spawn(self.process_send_queue()) await group.spawn(self.process_forward_queue()) except Exception as e: self.logger.exception("taskgroup died.") @@ -427,13 +427,13 @@ class OnionMessageManager(Logger): async def process_forward_queue(self): while True: - scheduled, expires, onion_packet, blinding, node_id = await self.forwardqueue.get() + scheduled, expires, onion_packet, blinding, node_id = await self.forward_queue.get() if expires <= now(): self.logger.debug(f'forward expired {node_id=}') continue if scheduled > now(): # return to queue - self.forwardqueue.put_nowait((scheduled, expires, onion_packet, blinding, node_id)) + self.forward_queue.put_nowait((scheduled, expires, onion_packet, blinding, node_id)) await asyncio.sleep(self.SLEEP_DELAY) # sleep here, as the first queue item wasn't due yet continue @@ -449,24 +449,24 @@ class OnionMessageManager(Logger): ) except BaseException as e: self.logger.debug(f'error while sending {node_id=} e={e!r}') - self.forwardqueue.put_nowait((now() + self.FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) + self.forward_queue.put_nowait((now() + self.FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) def submit_forward( self, *, onion_packet: OnionPacket, blinding: bytes, node_id: bytes): - if self.forwardqueue.qsize() >= self.FORWARD_MAX_QUEUE: + if self.forward_queue.qsize() >= self.FORWARD_MAX_QUEUE: self.logger.debug('forward queue full, dropping packet') return expires = now() + self.FORWARD_RETRY_TIMEOUT queueitem = (now(), expires, onion_packet, blinding, node_id) - self.forwardqueue.put_nowait(queueitem) + self.forward_queue.put_nowait(queueitem) - async def process_request_reply_queue(self): + async def process_send_queue(self): while True: - scheduled, expires, key = await self.requestreply_queue.get() - requestreply = self.get_requestreply(key) + scheduled, expires, key = await self.send_queue.get() + requestreply = self.get_pending_message(key) if requestreply is None: self.logger.debug(f'no data for key {key=}') continue @@ -475,32 +475,32 @@ class OnionMessageManager(Logger): continue if expires <= now(): self.logger.debug(f'expired {key=}') - self._set_requestreply_result(key, Timeout()) + self._set_message_result(key, Timeout()) continue if scheduled > now(): # return to queue self.logger.debug(f'return to queue {key=}, {scheduled - now()}') - self.requestreply_queue.put_nowait((scheduled, expires, key)) + self.send_queue.put_nowait((scheduled, expires, key)) await asyncio.sleep(self.SLEEP_DELAY) # sleep here, as the first queue item wasn't due yet continue try: - self._send_pending_requestreply(key) + self._send_pending_message(key) except BaseException as e: self.logger.debug(f'error while sending {key=} {e!r}') - self._set_requestreply_result(key, copy.copy(e)) + self._set_message_result(key, copy.copy(e)) # NOTE: above, when passing the caught exception instance e directly it leads to GeneratorExit() in if isinstance(e, NoRouteFound) and e.peer_address: await self.lnwallet.add_peer(str(e.peer_address)) else: self.logger.debug(f'resubmit {key=}') - self.requestreply_queue.put_nowait((now() + self.REQUEST_REPLY_RETRY_DELAY, expires, key)) + self.send_queue.put_nowait((now() + self.REQUEST_REPLY_RETRY_DELAY, expires, key)) - def get_requestreply(self, key): + def get_pending_message(self, key): with self.pending_lock: return self.pending.get(key) - def _set_requestreply_result(self, key, result): + def _set_message_result(self, key, result): with self.pending_lock: requestreply = self.pending.get(key) if requestreply is None: @@ -509,7 +509,7 @@ class OnionMessageManager(Logger): self.pending[key]['result'] = result requestreply['ev'].set() - def _remove_requestreply(self, key): + def _remove_pending_message(self, key): with self.pending_lock: requestreply = self.pending.get(key) if requestreply is None: @@ -517,7 +517,7 @@ class OnionMessageManager(Logger): requestreply['ev'].set() del self.pending[key] - def submit_requestreply( + def submit_send( self, *, payload: dict, node_id_or_blinded_path: bytes, @@ -533,7 +533,7 @@ class OnionMessageManager(Logger): key = os.urandom(8) assert type(key) is bytes and len(key) >= 8 - self.logger.debug(f'submit_requestreply {key=} {payload=} {node_id_or_blinded_path=}') + self.logger.debug(f'submit_send {key=} {payload=} {node_id_or_blinded_path=}') with self.pending_lock: if key in self.pending: @@ -547,12 +547,12 @@ class OnionMessageManager(Logger): # tuple = (when to process, when it expires, key) expires = now() + self.REQUEST_REPLY_TIMEOUT queueitem = (now(), expires, key) - self.requestreply_queue.put_nowait(queueitem) - task = asyncio.create_task(self._requestreply_task(key)) + self.send_queue.put_nowait(queueitem) + task = asyncio.create_task(self._wait_task(key)) return task - async def _requestreply_task(self, key): - requestreply = self.get_requestreply(key) + async def _wait_task(self, key): + requestreply = self.get_pending_message(key) assert requestreply if requestreply is None: return @@ -563,21 +563,21 @@ class OnionMessageManager(Logger): self.logger.debug(f'wait task end {key}') try: - requestreply = self.get_requestreply(key) + requestreply = self.get_pending_message(key) assert requestreply result = requestreply.get('result') if isinstance(result, Exception): raise result # raising in the task requires caller to explicitly extract exception. return result finally: - self._remove_requestreply(key) + self._remove_pending_message(key) - def _send_pending_requestreply(self, key): + def _send_pending_message(self, key): """adds reply_path to payload""" - data = self.get_requestreply(key) + data = self.get_pending_message(key) payload = data.get('payload') node_id_or_blinded_path = data.get('node_id_or_blinded_path') - self.logger.debug(f'send_requestreply {key=} {payload=} {node_id_or_blinded_path=}') + self.logger.debug(f'send_pending_message {key=} {payload=} {node_id_or_blinded_path=}') final_payload = copy.deepcopy(payload) @@ -624,12 +624,12 @@ class OnionMessageManager(Logger): self.logger.warning('not a reply to our request (unknown path_id prefix)') return key = correl_data[8:] - requestreply = self.get_requestreply(key) + requestreply = self.get_pending_message(key) if requestreply is None: self.logger.warning('not a reply to our request (unknown request)') return - self._set_requestreply_result(key, (recipient_data, payload)) + self._set_message_result(key, (recipient_data, payload)) def on_onion_message_received_unsolicited(self, recipient_data, payload): self.logger.debug('unsolicited onion_message received') diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 46bdc609a..13eee113a 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -307,7 +307,7 @@ class TestOnionMessageManager(ElectrumTestCase): self.eve_pub = self.eve.get_public_key_bytes(compressed=True) async def run_test1(self, t): - t1 = t.submit_requestreply( + t1 = t.submit_send( payload={'message': {'text': 'alice_timeout'.encode('utf-8')}}, node_id_or_blinded_path=self.alice_pub) @@ -315,7 +315,7 @@ class TestOnionMessageManager(ElectrumTestCase): await t1 async def run_test2(self, t): - t2 = t.submit_requestreply( + t2 = t.submit_send( payload={'message': {'text': 'bob_slow_timeout'.encode('utf-8')}}, node_id_or_blinded_path=self.bob_pub) @@ -323,7 +323,7 @@ class TestOnionMessageManager(ElectrumTestCase): await t2 async def run_test3(self, t, rkey): - t3 = t.submit_requestreply( + t3 = t.submit_send( payload={'message': {'text': 'carol_with_immediate_reply'.encode('utf-8')}}, node_id_or_blinded_path=self.carol_pub, key=rkey) @@ -332,7 +332,7 @@ class TestOnionMessageManager(ElectrumTestCase): self.assertEqual(t3_result, ({'path_id': {'data': b'electrum' + rkey}}, {})) async def run_test4(self, t, rkey): - t4 = t.submit_requestreply( + t4 = t.submit_send( payload={'message': {'text': 'dave_with_slow_reply'.encode('utf-8')}}, node_id_or_blinded_path=self.dave_pub, key=rkey) @@ -341,7 +341,7 @@ class TestOnionMessageManager(ElectrumTestCase): self.assertEqual(t4_result, ({'path_id': {'data': b'electrum' + rkey}}, {})) async def run_test5(self, t): - t5 = t.submit_requestreply( + t5 = t.submit_send( payload={'message': {'text': 'no_peer'.encode('utf-8')}}, node_id_or_blinded_path=self.eve_pub) From 71b97619812ee510a5353112fad93e00b98495e2 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sat, 18 Jan 2025 14:01:02 +0100 Subject: [PATCH 13/22] onion_messages_manager: - use namedtuple instead of dict for pending messages - use asyncio.Future instead of event and result --- electrum/onion_message.py | 99 ++++++++++++++------------------------- 1 file changed, 34 insertions(+), 65 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 64f89927f..c9ab88a53 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -28,7 +28,7 @@ import threading import time from random import random -from typing import TYPE_CHECKING, Optional, List, Sequence +from typing import TYPE_CHECKING, Optional, List, Sequence, NamedTuple import electrum_ecc as ecc @@ -40,7 +40,7 @@ from electrum.lnonion import (get_bolt04_onion_key, OnionPacket, process_onion_p OnionHopsDataSingle, decrypt_onionmsg_data_tlv, encrypt_onionmsg_data_tlv, get_shared_secrets_along_route, new_onion_packet) from electrum.lnutil import LnFeatures -from electrum.util import OldTaskGroup +from electrum.util import OldTaskGroup, log_exceptions # do not use util.now, because it rounds to integers def now(): @@ -380,6 +380,11 @@ def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, class Timeout(Exception): pass +class OnionMessageRequest(NamedTuple): + future: asyncio.Future + payload: bytes + node_id_or_blinded_path: bytes + class OnionMessageManager(Logger): """handle state around onion message sends and receives @@ -398,7 +403,6 @@ class OnionMessageManager(Logger): self.network = None # type: Optional['Network'] self.taskgroup = None # type: OldTaskGroup self.lnwallet = lnwallet - self.pending = {} self.pending_lock = threading.Lock() self.send_queue = asyncio.PriorityQueue() @@ -411,16 +415,13 @@ class OnionMessageManager(Logger): self.taskgroup = OldTaskGroup() asyncio.run_coroutine_threadsafe(self.main_loop(), self.network.asyncio_loop) + @log_exceptions async def main_loop(self): self.logger.info("starting taskgroup.") - try: - async with self.taskgroup as group: - await group.spawn(self.process_send_queue()) - await group.spawn(self.process_forward_queue()) - except Exception as e: - self.logger.exception("taskgroup died.") - else: - self.logger.info("taskgroup stopped.") + async with self.taskgroup as group: + await group.spawn(self.process_send_queue()) + await group.spawn(self.process_forward_queue()) + self.logger.info("taskgroup stopped.") async def stop(self): await self.taskgroup.cancel_remaining() @@ -466,16 +467,16 @@ class OnionMessageManager(Logger): async def process_send_queue(self): while True: scheduled, expires, key = await self.send_queue.get() - requestreply = self.get_pending_message(key) - if requestreply is None: + req = self.pending.get(key) + if req is None: self.logger.debug(f'no data for key {key=}') continue - if requestreply.get('result') is not None: + if req.future.done(): self.logger.debug(f'has result! {key=}') continue if expires <= now(): self.logger.debug(f'expired {key=}') - self._set_message_result(key, Timeout()) + req.future.set_exception(Timeout()) continue if scheduled > now(): # return to queue @@ -483,12 +484,11 @@ class OnionMessageManager(Logger): self.send_queue.put_nowait((scheduled, expires, key)) await asyncio.sleep(self.SLEEP_DELAY) # sleep here, as the first queue item wasn't due yet continue - try: self._send_pending_message(key) except BaseException as e: self.logger.debug(f'error while sending {key=} {e!r}') - self._set_message_result(key, copy.copy(e)) + req.future.set_exception(copy.copy(e)) # NOTE: above, when passing the caught exception instance e directly it leads to GeneratorExit() in if isinstance(e, NoRouteFound) and e.peer_address: await self.lnwallet.add_peer(str(e.peer_address)) @@ -496,26 +496,10 @@ class OnionMessageManager(Logger): self.logger.debug(f'resubmit {key=}') self.send_queue.put_nowait((now() + self.REQUEST_REPLY_RETRY_DELAY, expires, key)) - def get_pending_message(self, key): - with self.pending_lock: - return self.pending.get(key) - - def _set_message_result(self, key, result): - with self.pending_lock: - requestreply = self.pending.get(key) - if requestreply is None: - self.logger.error(f'requestreply with {key=} not found!') - return - self.pending[key]['result'] = result - requestreply['ev'].set() - def _remove_pending_message(self, key): with self.pending_lock: - requestreply = self.pending.get(key) - if requestreply is None: - return - requestreply['ev'].set() - del self.pending[key] + if key in self.pending: + del self.pending[key] def submit_send( self, *, @@ -535,48 +519,34 @@ class OnionMessageManager(Logger): self.logger.debug(f'submit_send {key=} {payload=} {node_id_or_blinded_path=}') + req = OnionMessageRequest( + future=asyncio.Future(), + payload=payload, + node_id_or_blinded_path=node_id_or_blinded_path + ) with self.pending_lock: if key in self.pending: raise Exception(f'{key=} already exists!') - self.pending[key] = { - 'ev': asyncio.Event(), - 'payload': payload, - 'node_id_or_blinded_path': node_id_or_blinded_path - } + self.pending[key] = req # tuple = (when to process, when it expires, key) expires = now() + self.REQUEST_REPLY_TIMEOUT queueitem = (now(), expires, key) self.send_queue.put_nowait(queueitem) - task = asyncio.create_task(self._wait_task(key)) + task = asyncio.create_task(self._wait_task(key, req.future)) return task - async def _wait_task(self, key): - requestreply = self.get_pending_message(key) - assert requestreply - if requestreply is None: - return + async def _wait_task(self, key, future): try: - self.logger.debug(f'wait task start {key}') - await requestreply['ev'].wait() - finally: - self.logger.debug(f'wait task end {key}') - - try: - requestreply = self.get_pending_message(key) - assert requestreply - result = requestreply.get('result') - if isinstance(result, Exception): - raise result # raising in the task requires caller to explicitly extract exception. - return result + return await future finally: self._remove_pending_message(key) def _send_pending_message(self, key): """adds reply_path to payload""" - data = self.get_pending_message(key) - payload = data.get('payload') - node_id_or_blinded_path = data.get('node_id_or_blinded_path') + req = self.pending.get(key) + payload = req.payload + node_id_or_blinded_path = req.node_id_or_blinded_path self.logger.debug(f'send_pending_message {key=} {payload=} {node_id_or_blinded_path=}') final_payload = copy.deepcopy(payload) @@ -624,12 +594,11 @@ class OnionMessageManager(Logger): self.logger.warning('not a reply to our request (unknown path_id prefix)') return key = correl_data[8:] - requestreply = self.get_pending_message(key) - if requestreply is None: + req = self.pending.get(key) + if req is None: self.logger.warning('not a reply to our request (unknown request)') return - - self._set_message_result(key, (recipient_data, payload)) + req.future.set_result((recipient_data, payload)) def on_onion_message_received_unsolicited(self, recipient_data, payload): self.logger.debug('unsolicited onion_message received') From e216f1b324df8ee5fa792ac20b7456a54294400b Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Wed, 12 Feb 2025 14:00:18 +0100 Subject: [PATCH 14/22] onion_messages: add parameter typing and comments --- electrum/onion_message.py | 105 +++++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 30 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index c9ab88a53..6e1aa322c 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -42,6 +42,7 @@ from electrum.lnonion import (get_bolt04_onion_key, OnionPacket, process_onion_p from electrum.lnutil import LnFeatures from electrum.util import OldTaskGroup, log_exceptions + # do not use util.now, because it rounds to integers def now(): return time.time() @@ -66,9 +67,14 @@ class NoRouteFound(Exception): self.peer_address = peer_address -def create_blinded_path(session_key: bytes, path: List[bytes], final_recipient_data: dict, *, - hop_extras: Optional[Sequence[dict]] = None, - dummy_hops: Optional[int] = 0) -> dict: +def create_blinded_path( + session_key: bytes, + path: List[bytes], + final_recipient_data: dict, + *, + hop_extras: Optional[Sequence[dict]] = None, + dummy_hops: Optional[int] = 0 +) -> dict: # dummy hops could be inserted anywhere in the path, but for compatibility just add them at the end # because blinded paths are usually constructed towards ourselves, and we know we can handle dummy hops. if dummy_hops: @@ -114,7 +120,7 @@ def create_blinded_path(session_key: bytes, path: List[bytes], final_recipient_d return blinded_path -def blinding_privkey(privkey, blinding): +def blinding_privkey(privkey: bytes, blinding: bytes) -> bytes: shared_secret = get_ecdh(privkey, blinding) b_hmac = get_bolt04_onion_key(b'blinded_node_id', shared_secret) b_hmac_int = int.from_bytes(b_hmac, byteorder="big") @@ -126,13 +132,13 @@ def blinding_privkey(privkey, blinding): return our_privkey -def is_onion_message_node(node_id: bytes, node_info: Optional['NodeInfo']): +def is_onion_message_node(node_id: bytes, node_info: Optional['NodeInfo']) -> bool: if not node_info: return False return LnFeatures(node_info.features).supports(LnFeatures.OPTION_ONION_MESSAGE_OPT) -def encrypt_onionmsg_tlv_hops_data(hops_data, hop_shared_secrets): +def encrypt_onionmsg_tlv_hops_data(hops_data: List[OnionHopsDataSingle], hop_shared_secrets: List[bytes]) -> None: """encrypt unencrypted onionmsg_tlv.encrypted_recipient_data for hops with blind_fields""" num_hops = len(hops_data) for i in range(num_hops): @@ -175,7 +181,12 @@ def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> List[ raise NoRouteFound('no path found') -def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: bytes, destination_payload: dict, session_key: bytes = None): +def send_onion_message_to( + lnwallet: 'LNWallet', + node_id_or_blinded_path: bytes, + destination_payload: dict, + session_key: bytes = None +) -> None: if session_key is None: session_key = os.urandom(32) @@ -350,9 +361,19 @@ def send_onion_message_to(lnwallet: 'LNWallet', node_id_or_blinded_path: bytes, ) -def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, - max_paths: int = REQUEST_REPLY_PATHS_MAX, - preferred_node_id: bytes = None) -> List[dict]: +def get_blinded_reply_paths( + lnwallet: 'LNWallet', + path_id: bytes, + *, + max_paths: int = REQUEST_REPLY_PATHS_MAX, + preferred_node_id: bytes = None +) -> List[dict]: + """construct a list of blinded reply_paths. + current logic: + - uses current onion_message capable channel peers if exist + - otherwise, uses current onion_message capable peers + - prefers preferred_node_id if given + - reply_path introduction points are direct peers only (TODO: longer reply paths)""" # TODO: build longer paths and/or add dummy hops to increase privacy my_active_channels = [chan for chan in lnwallet.channels.values() if chan.is_active()] my_onionmsg_channels = [chan for chan in my_active_channels if lnwallet.peers.get(chan.node_id) and @@ -361,7 +382,7 @@ def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, result = [] mynodeid = lnwallet.node_keypair.pubkey - mydata = {'path_id': {'data': path_id}} # same used in every path + mydata = {'path_id': {'data': path_id}} # same path_id used in every reply path if len(my_onionmsg_channels): # randomize list, but prefer preferred_node_id rchans = sorted(my_onionmsg_channels, key=lambda x: random() if x.node_id != preferred_node_id else 0) @@ -380,6 +401,7 @@ def get_blinded_reply_paths(lnwallet: 'LNWallet', path_id: bytes, *, class Timeout(Exception): pass + class OnionMessageRequest(NamedTuple): future: asyncio.Future payload: bytes @@ -387,9 +409,17 @@ class OnionMessageRequest(NamedTuple): class OnionMessageManager(Logger): - """handle state around onion message sends and receives + """handle state around onion message sends and receives. + - one instance per (ln)wallet - association between onion message and their replies - - manage re-send attempts, TODO: iterate through routes (both directions)""" + - manage re-send attempts while iterating over possible routes. Onion messages are unreliable + and fail silently if they don't reach their destination (or the reply gets dropped along the route back), + so the BOLT-4 spec suggests to send multiple messages, each with a different route to the introduction point). + - forwards are best-effort. They should not need retrying, but a queue is used to limit the pacing of forwarding, + and limiting the number of outstanding forwards. Any onion message forwards arriving when the forward queue + is full will be dropped. + + TODO: iterate through routes for each request""" SLEEP_DELAY = 1 REQUEST_REPLY_TIMEOUT = 30 @@ -403,12 +433,12 @@ class OnionMessageManager(Logger): self.network = None # type: Optional['Network'] self.taskgroup = None # type: OldTaskGroup self.lnwallet = lnwallet - self.pending = {} + self.pending = {} # type: dict[bytes, OnionMessageRequest] self.pending_lock = threading.Lock() self.send_queue = asyncio.PriorityQueue() self.forward_queue = asyncio.PriorityQueue() - def start_network(self, *, network: 'Network'): + def start_network(self, *, network: 'Network') -> None: assert network assert self.network is None, "already started" self.network = network @@ -416,17 +446,17 @@ class OnionMessageManager(Logger): asyncio.run_coroutine_threadsafe(self.main_loop(), self.network.asyncio_loop) @log_exceptions - async def main_loop(self): + async def main_loop(self) -> None: self.logger.info("starting taskgroup.") async with self.taskgroup as group: await group.spawn(self.process_send_queue()) await group.spawn(self.process_forward_queue()) self.logger.info("taskgroup stopped.") - async def stop(self): + async def stop(self) -> None: await self.taskgroup.cancel_remaining() - async def process_forward_queue(self): + async def process_forward_queue(self) -> None: while True: scheduled, expires, onion_packet, blinding, node_id = await self.forward_queue.get() if expires <= now(): @@ -450,13 +480,14 @@ class OnionMessageManager(Logger): ) except BaseException as e: self.logger.debug(f'error while sending {node_id=} e={e!r}') + # TODO: it is debatable whether we want to retry a forward. self.forward_queue.put_nowait((now() + self.FORWARD_RETRY_DELAY, expires, onion_packet, blinding, node_id)) def submit_forward( self, *, onion_packet: OnionPacket, blinding: bytes, - node_id: bytes): + node_id: bytes) -> None: if self.forward_queue.qsize() >= self.FORWARD_MAX_QUEUE: self.logger.debug('forward queue full, dropping packet') return @@ -464,7 +495,7 @@ class OnionMessageManager(Logger): queueitem = (now(), expires, onion_packet, blinding, node_id) self.forward_queue.put_nowait(queueitem) - async def process_send_queue(self): + async def process_send_queue(self) -> None: while True: scheduled, expires, key = await self.send_queue.get() req = self.pending.get(key) @@ -496,7 +527,7 @@ class OnionMessageManager(Logger): self.logger.debug(f'resubmit {key=}') self.send_queue.put_nowait((now() + self.REQUEST_REPLY_RETRY_DELAY, expires, key)) - def _remove_pending_message(self, key): + def _remove_pending_message(self, key: bytes) -> None: with self.pending_lock: if key in self.pending: del self.pending[key] @@ -536,13 +567,13 @@ class OnionMessageManager(Logger): task = asyncio.create_task(self._wait_task(key, req.future)) return task - async def _wait_task(self, key, future): + async def _wait_task(self, key: bytes, future: asyncio.Future): try: return await future finally: self._remove_pending_message(key) - def _send_pending_message(self, key): + def _send_pending_message(self, key: bytes) -> None: """adds reply_path to payload""" req = self.pending.get(key) payload = req.payload @@ -568,7 +599,7 @@ class OnionMessageManager(Logger): # TODO: use payload to determine prefix? return b'electrum' + key - def on_onion_message_received(self, recipient_data, payload): + def on_onion_message_received(self, recipient_data: dict, payload: dict) -> None: # we are destination, sanity checks # - if `encrypted_data_tlv` contains `allowed_features`: # - MUST ignore the message if: @@ -587,7 +618,7 @@ class OnionMessageManager(Logger): else: self.on_onion_message_received_reply(recipient_data, payload) - def on_onion_message_received_reply(self, recipient_data, payload): + def on_onion_message_received_reply(self, recipient_data: dict, payload: dict) -> None: # check if this reply is associated with a known request correl_data = recipient_data['path_id'].get('data') if not correl_data[:8] == b'electrum': @@ -600,11 +631,18 @@ class OnionMessageManager(Logger): return req.future.set_result((recipient_data, payload)) - def on_onion_message_received_unsolicited(self, recipient_data, payload): + def on_onion_message_received_unsolicited(self, recipient_data: dict, payload: dict) -> None: self.logger.debug('unsolicited onion_message received') self.logger.debug(f'payload: {payload!r}') - # TODO: currently only accepts simple text 'message' payload. + # This func currently only accepts simple text 'message' payload, a.k.a 'unknown_tag_1' + # in the bolt-4 test vectors. + # + # TODO: for BOLT-12, handle invoice_request here, which should correspond with a previously generated Offer. + # as this is not strictly part of BOLT-4, we should probably create a registration mechanism + # for various types of payloads, so we can let external code plug into onion messages + # e.g. via a decorator, something like + # @onion_message_request_handler(payload_key='invoice_request') for BOLT12 invoice requests. if 'message' not in payload: self.logger.error('Unsupported onion message payload') @@ -622,7 +660,13 @@ class OnionMessageManager(Logger): self.logger.info(f'onion message with text received: {text}') - def on_onion_message_forward(self, recipient_data, onion_packet, blinding, shared_secret): + def on_onion_message_forward( + self, + recipient_data: dict, + onion_packet: OnionPacket, + blinding: bytes, + shared_secret: bytes + ) -> None: if recipient_data.get('path_id'): self.logger.error('cannot forward onion_message, path_id in encrypted_data_tlv') return @@ -661,7 +705,8 @@ class OnionMessageManager(Logger): self.submit_forward(onion_packet=onion_packet, blinding=next_blinding, node_id=next_node_id) - def on_onion_message(self, payload): + def on_onion_message(self, payload: dict) -> None: + """handle arriving onion_message.""" blinding = payload.get('blinding') if not blinding: self.logger.error('missing blinding') @@ -676,7 +721,7 @@ class OnionMessageManager(Logger): onion_packet = OnionPacket.from_bytes(packet) self.process_onion_message_packet(blinding, onion_packet) - def process_onion_message_packet(self, blinding: bytes, onion_packet: OnionPacket): + def process_onion_message_packet(self, blinding: bytes, onion_packet: OnionPacket) -> None: our_privkey = blinding_privkey(self.lnwallet.node_keypair.privkey, blinding) processed_onion_packet = process_onion_packet(onion_packet, our_privkey, tlv_stream_name='onionmsg_tlv') payload = processed_onion_packet.hop_data.payload From 6e35ffe4b56d2a3d6d35f91acdedea9a9a8cb243 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Wed, 12 Feb 2025 14:28:30 +0100 Subject: [PATCH 15/22] lnmsg: support both primitive and complex types (subtypes) in LNSerializer. This renames lnmsg._{read,write}_field to lnmsg._{read,write}_primitive_field, renames LNSerializer._{read,write}_complex_type to LNSerializer.{read,write}_field and allows LNSerializer.{read,write}_field to handle both primitive and complex types. Also makes these funcs public, as these encodings are used outside of lnmsg as well (e.g. encoding blinded paths in BOLT12 invoice_request) --- electrum/commands.py | 2 +- electrum/lnmsg.py | 134 ++++++++++++++++++------------------ electrum/onion_message.py | 2 +- tests/test_onion_message.py | 2 +- 4 files changed, 69 insertions(+), 71 deletions(-) diff --git a/electrum/commands.py b/electrum/commands.py index 6e51fdcdd..e1ccf3a29 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -1516,7 +1516,7 @@ class Commands(Logger): blinded_path = create_blinded_path(session_key, path=path, final_recipient_data={}, dummy_hops=dummy_hops) with io.BytesIO() as blinded_path_fd: - OnionWireSerializer._write_complex_field( + OnionWireSerializer.write_field( fd=blinded_path_fd, field_type='blinded_path', count=1, diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index da849f22f..eb0b761e2 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -88,8 +88,14 @@ def read_bigsize_int(fd: io.BytesIO) -> Optional[int]: # TODO: maybe if field_type is not "byte", we could return a list of type_len sized chunks? # if field_type is a numeric, we could return a list of ints? -def _read_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str]) -> Union[bytes, int]: - if not fd: raise Exception() +def _read_primitive_field( + *, + fd: io.BytesIO, + field_type: str, + count: Union[int, str] +) -> Union[bytes, int]: + if not fd: + raise Exception() if isinstance(count, int): assert count >= 0, f"{count!r} must be non-neg int" elif count == "...": @@ -174,9 +180,15 @@ def _read_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str]) -> U # TODO: maybe for "value" we could accept a list with len "count" of appropriate items -def _write_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str], - value: Union[bytes, int]) -> None: - if not fd: raise Exception() +def _write_primitive_field( + *, + fd: io.BytesIO, + field_type: str, + count: Union[int, str], + value: Union[bytes, int] +) -> None: + if not fd: + raise Exception() if isinstance(count, int): assert count >= 0, f"{count!r} must be non-neg int" elif count == "...": @@ -263,18 +275,18 @@ def _write_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str], def _read_tlv_record(*, fd: io.BytesIO) -> Tuple[int, bytes]: if not fd: raise Exception() - tlv_type = _read_field(fd=fd, field_type="bigsize", count=1) - tlv_len = _read_field(fd=fd, field_type="bigsize", count=1) - tlv_val = _read_field(fd=fd, field_type="byte", count=tlv_len) + tlv_type = _read_primitive_field(fd=fd, field_type="bigsize", count=1) + tlv_len = _read_primitive_field(fd=fd, field_type="bigsize", count=1) + tlv_val = _read_primitive_field(fd=fd, field_type="byte", count=tlv_len) return tlv_type, tlv_val def _write_tlv_record(*, fd: io.BytesIO, tlv_type: int, tlv_val: bytes) -> None: if not fd: raise Exception() tlv_len = len(tlv_val) - _write_field(fd=fd, field_type="bigsize", count=1, value=tlv_type) - _write_field(fd=fd, field_type="bigsize", count=1, value=tlv_len) - _write_field(fd=fd, field_type="byte", count=tlv_len, value=tlv_val) + _write_primitive_field(fd=fd, field_type="bigsize", count=1, value=tlv_type) + _write_primitive_field(fd=fd, field_type="bigsize", count=1, value=tlv_len) + _write_primitive_field(fd=fd, field_type="byte", count=tlv_len, value=tlv_val) def _resolve_field_count(field_count_str: str, *, vars_dict: dict, allow_any=False) -> Union[int, str]: @@ -385,10 +397,19 @@ class LNSerializer: else: pass # TODO - def _write_complex_field(self, *, fd: io.BytesIO, field_type: str, count: Union[int, str], - value: Union[List[Dict[str, Any]], Dict[str, Any]]) -> None: + def write_field( + self, + *, + fd: io.BytesIO, + field_type: str, + count: Union[int, str], + value: Union[List[Dict[str, Any]], Dict[str, Any]] + ) -> None: assert fd - assert field_type in self.subtypes, f"unknown subtype {field_type}" + + if field_type not in self.subtypes: + _write_primitive_field(fd=fd, field_type=field_type, count=count, value=value) + return if isinstance(count, int): assert count >= 0, f"{count!r} must be non-neg int" @@ -428,22 +449,24 @@ class LNSerializer: if subtype_field_name not in record: raise Exception(f'complex field type {field_type} missing element {subtype_field_name}') - if subtype_field_type in self.subtypes: - self._write_complex_field( - fd=fd, - field_type=subtype_field_type, - count=subtype_field_count, - value=record[subtype_field_name]) - else: - _write_field( - fd=fd, - field_type=subtype_field_type, - count=subtype_field_count, - value=record[subtype_field_name]) + self.write_field( + fd=fd, + field_type=subtype_field_type, + count=subtype_field_count, + value=record[subtype_field_name]) - def _read_complex_field(self, *, fd: io.BytesIO, field_type: str, count: Union[int, str])\ - -> Union[bytes, List[Dict[str, Any]], Dict[str, Any]]: + def read_field( + self, + *, + fd: io.BytesIO, + field_type: str, + count: Union[int, str] + ) -> Union[bytes, List[Dict[str, Any]], Dict[str, Any]]: assert fd + + if field_type not in self.subtypes: + return _read_primitive_field(fd=fd, field_type=field_type, count=count) + if isinstance(count, int): assert count >= 0, f"{count!r} must be non-neg int" elif count == "...": @@ -468,16 +491,10 @@ class LNSerializer: vars_dict=parsed, allow_any=True) - if subtype_field_type in self.subtypes: - parsed[subtype_field_name] = self._read_complex_field( - fd=fd, - field_type=subtype_field_type, - count=subtype_field_count) - else: - parsed[subtype_field_name] = _read_field( - fd=fd, - field_type=subtype_field_type, - count=subtype_field_count) + parsed[subtype_field_name] = self.read_field( + fd=fd, + field_type=subtype_field_type, + count=subtype_field_count) parsedlist.append(parsed) return parsedlist if count == '...' or count > 1 else parsedlist[0] @@ -503,18 +520,11 @@ class LNSerializer: vars_dict=kwargs[tlv_record_name], allow_any=True) field_value = kwargs[tlv_record_name][field_name] - if field_type in self.subtypes: - self._write_complex_field( - fd=tlv_record_fd, - field_type=field_type, - count=field_count, - value=field_value) - else: - _write_field( - fd=tlv_record_fd, - field_type=field_type, - count=field_count, - value=field_value) + self.write_field( + fd=tlv_record_fd, + field_type=field_type, + count=field_count, + value=field_value) else: raise Exception(f"unexpected row in scheme: {row!r}") _write_tlv_record(fd=fd, tlv_type=tlv_record_type, tlv_val=tlv_record_fd.getvalue()) @@ -557,16 +567,10 @@ class LNSerializer: vars_dict=parsed[tlv_record_name], allow_any=True) #print(f">> count={field_count}. parsed={parsed}") - if field_type in self.subtypes: - parsed[tlv_record_name][field_name] = self._read_complex_field( - fd=tlv_record_fd, - field_type=field_type, - count=field_count) - else: - parsed[tlv_record_name][field_name] = _read_field( - fd=tlv_record_fd, - field_type=field_type, - count=field_count) + parsed[tlv_record_name][field_name] = self.read_field( + fd=tlv_record_fd, + field_type=field_type, + count=field_count) else: raise Exception(f"unexpected row in scheme: {row!r}") if _num_remaining_bytes_to_read(tlv_record_fd) > 0: @@ -603,10 +607,7 @@ class LNSerializer: except KeyError: field_value = 0 # default mandatory fields to zero #print(f">>> encode_msg. writing field: {field_name}. value={field_value!r}. field_type={field_type!r}. count={field_count!r}") - _write_field(fd=fd, - field_type=field_type, - count=field_count, - value=field_value) + _write_primitive_field(fd=fd, field_type=field_type, count=field_count, value=field_value) #print(f">>> encode_msg. so far: {fd.getvalue().hex()}") else: raise Exception(f"unexpected row in scheme: {row!r}") @@ -651,10 +652,7 @@ class LNSerializer: parsed[tlv_stream_name] = d continue #print(f">> count={field_count}. parsed={parsed}") - parsed[field_name] = _read_field( - fd=fd, - field_type=field_type, - count=field_count) + parsed[field_name] = _read_primitive_field(fd=fd, field_type=field_type, count=field_count) else: raise Exception(f"unexpected row in scheme: {row!r}") except FailedToParseMsg as e: diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 6e1aa322c..3ad2cc93a 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -193,7 +193,7 @@ def send_onion_message_to( if len(node_id_or_blinded_path) > 33: # assume blinded path with io.BytesIO(node_id_or_blinded_path) as blinded_path_fd: try: - blinded_path = OnionWireSerializer._read_complex_field( + blinded_path = OnionWireSerializer.read_field( fd=blinded_path_fd, field_type='blinded_path', count=1) diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 13eee113a..d855d27db 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -194,7 +194,7 @@ class TestOnionMessage(ElectrumTestCase): # TODO: serialization test to test_lnmsg.py with io.BytesIO() as blinded_path_fd: - OnionWireSerializer._write_complex_field( + OnionWireSerializer.write_field( fd=blinded_path_fd, field_type='blinded_path', count=1, From f9a374729e9ba9e329ced65f80bbebe6a93f1836 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Wed, 12 Feb 2025 15:35:01 +0100 Subject: [PATCH 16/22] onion_messages: generalize List parameter typing to more abstract Sequence --- electrum/onion_message.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 3ad2cc93a..93991759c 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -28,7 +28,7 @@ import threading import time from random import random -from typing import TYPE_CHECKING, Optional, List, Sequence, NamedTuple +from typing import TYPE_CHECKING, Optional, Sequence, NamedTuple import electrum_ecc as ecc @@ -69,7 +69,7 @@ class NoRouteFound(Exception): def create_blinded_path( session_key: bytes, - path: List[bytes], + path: Sequence[bytes], final_recipient_data: dict, *, hop_extras: Optional[Sequence[dict]] = None, @@ -138,7 +138,10 @@ def is_onion_message_node(node_id: bytes, node_info: Optional['NodeInfo']) -> bo return LnFeatures(node_info.features).supports(LnFeatures.OPTION_ONION_MESSAGE_OPT) -def encrypt_onionmsg_tlv_hops_data(hops_data: List[OnionHopsDataSingle], hop_shared_secrets: List[bytes]) -> None: +def encrypt_onionmsg_tlv_hops_data( + hops_data: Sequence[OnionHopsDataSingle], + hop_shared_secrets: Sequence[bytes] +) -> None: """encrypt unencrypted onionmsg_tlv.encrypted_recipient_data for hops with blind_fields""" num_hops = len(hops_data) for i in range(num_hops): @@ -148,7 +151,7 @@ def encrypt_onionmsg_tlv_hops_data(hops_data: List[OnionHopsDataSingle], hop_sha hops_data[i].payload['encrypted_recipient_data'] = {'encrypted_recipient_data': encrypted_recipient_data} -def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> List[PathEdge]: +def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> Sequence[PathEdge]: """Constructs a route to the destination node_id, first by starting with peers with existing channels, and if no route found, opening a direct peer connection if node_id is found with an address in channel_db.""" @@ -367,7 +370,7 @@ def get_blinded_reply_paths( *, max_paths: int = REQUEST_REPLY_PATHS_MAX, preferred_node_id: bytes = None -) -> List[dict]: +) -> Sequence[dict]: """construct a list of blinded reply_paths. current logic: - uses current onion_message capable channel peers if exist From 560b244b4d02752467c012e3ce81f009549a7943 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Wed, 12 Feb 2025 15:35:38 +0100 Subject: [PATCH 17/22] onion_messages: replace more hardcoded test vector values with symbols --- tests/test_onion_message.py | 59 ++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index d855d27db..4db5fad10 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -35,13 +35,19 @@ OnionMessageManager.FORWARD_RETRY_DELAY *= TIME_STEP path = os.path.join(os.path.dirname(__file__), 'blinded-onion-message-onion-test.json') test_vectors = read_json_file(path) ONION_MESSAGE_PACKET = bfh(test_vectors['onionmessage']['onion_message_packet']) -ALICE_PUBKEY = bfh(test_vectors['route']['introduction_node_id']) -BOB_PUBKEY = bfh(test_vectors['generate']['hops'][0]['tlvs']['next_node_id']) -CAROL_PUBKEY = bfh(test_vectors['generate']['hops'][1]['tlvs']['next_node_id']) -DAVE_PUBKEY = bfh(test_vectors['generate']['hops'][2]['tlvs']['next_node_id']) +HOPS = test_vectors['generate']['hops'] +ALICE_TLVS = HOPS[0]['tlvs'] +BOB_TLVS = HOPS[1]['tlvs'] +CAROL_TLVS = HOPS[2]['tlvs'] +DAVE_TLVS = HOPS[3]['tlvs'] -BLINDING_SECRET = bfh(test_vectors['generate']['hops'][0]['blinding_secret']) -BLINDING_OVERRIDE_SECRET = bfh(test_vectors['generate']['hops'][0]['tlvs']['blinding_override_secret']) +ALICE_PUBKEY = bfh(test_vectors['route']['introduction_node_id']) +BOB_PUBKEY = bfh(ALICE_TLVS['next_node_id']) +CAROL_PUBKEY = bfh(BOB_TLVS['next_node_id']) +DAVE_PUBKEY = bfh(CAROL_TLVS['next_node_id']) + +BLINDING_SECRET = bfh(HOPS[0]['blinding_secret']) +BLINDING_OVERRIDE_SECRET = bfh(ALICE_TLVS['blinding_override_secret']) SESSION_KEY = bfh(test_vectors['generate']['session_key']) @@ -56,40 +62,39 @@ class TestOnionMessage(ElectrumTestCase): blinded_node_ids = blinded_node_ids1 + blinded_node_ids2 for i, ss in enumerate(hop_shared_secrets): - self.assertEqual(ss, bfh(test_vectors['generate']['hops'][i]['ss'])) + self.assertEqual(ss, bfh(HOPS[i]['ss'])) for i, ss in enumerate(blinded_node_ids): - self.assertEqual(ss, bfh(test_vectors['generate']['hops'][i]['blinded_node_id'])) + self.assertEqual(ss, bfh(HOPS[i]['blinded_node_id'])) - hops_tlvs = [x['tlvs'] for x in test_vectors['generate']['hops']] hops_data = [ OnionHopsDataSingle( tlv_stream_name='onionmsg_tlv', blind_fields={ - 'next_node_id': {'node_id': bfh(hops_tlvs[0]['next_node_id'])}, - 'next_blinding_override': {'blinding': bfh(hops_tlvs[0]['next_blinding_override'])}, + 'next_node_id': {'node_id': bfh(ALICE_TLVS['next_node_id'])}, + 'next_blinding_override': {'blinding': bfh(ALICE_TLVS['next_blinding_override'])}, } ), OnionHopsDataSingle( tlv_stream_name='onionmsg_tlv', blind_fields={ - 'next_node_id': {'node_id': bfh(hops_tlvs[1]['next_node_id'])}, - 'unknown_tag_561': {'data': bfh(hops_tlvs[1]['unknown_tag_561'])}, + 'next_node_id': {'node_id': bfh(BOB_TLVS['next_node_id'])}, + 'unknown_tag_561': {'data': bfh(BOB_TLVS['unknown_tag_561'])}, } ), OnionHopsDataSingle( tlv_stream_name='onionmsg_tlv', blind_fields={ - 'padding': {'padding': bfh(hops_tlvs[2]['padding'])}, - 'next_node_id': {'node_id': bfh(hops_tlvs[2]['next_node_id'])}, + 'padding': {'padding': bfh(CAROL_TLVS['padding'])}, + 'next_node_id': {'node_id': bfh(CAROL_TLVS['next_node_id'])}, } ), OnionHopsDataSingle( tlv_stream_name='onionmsg_tlv', - payload={'message': {'text': b'hello'}}, + payload={'message': {'text': bfh(test_vectors['onionmessage']['unknown_tag_1'])}}, blind_fields={ - 'padding': {'padding': bfh(hops_tlvs[3]['padding'])}, - 'path_id': {'data': bfh(hops_tlvs[3]['path_id'])}, - 'unknown_tag_65535': {'data': bfh(hops_tlvs[3]['unknown_tag_65535'])}, + 'padding': {'padding': bfh(DAVE_TLVS['padding'])}, + 'path_id': {'data': bfh(DAVE_TLVS['path_id'])}, + 'unknown_tag_65535': {'data': bfh(DAVE_TLVS['unknown_tag_65535'])}, } ) ] @@ -167,7 +172,7 @@ class TestOnionMessage(ElectrumTestCase): msgtype, data = decode_msg(bfh(onion_message_bob)) self.assertEqual(msgtype, 'onion_message') self.assertEqual(data, { - 'blinding': bfh(test_vectors['generate']['hops'][0]['tlvs']['next_blinding_override']), + 'blinding': bfh(ALICE_TLVS['next_blinding_override']), 'len': 1366, 'onion_message_packet': p.next_packet.to_bytes(), }) @@ -204,13 +209,13 @@ class TestOnionMessage(ElectrumTestCase): def prepare_blinded_path_bob_to_dave(self): final_recipient_data = { - 'padding': {'padding': b''}, - 'path_id': {'data': bfh('deadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0ffeedeadbeefbadc0')}, - 'unknown_tag_65535': {'data': bfh('06c1')} + 'padding': {'padding': bfh(DAVE_TLVS['padding'])}, + 'path_id': {'data': bfh(DAVE_TLVS['path_id'])}, + 'unknown_tag_65535': {'data': bfh(DAVE_TLVS['unknown_tag_65535'])} } hop_extras = [ - {'unknown_tag_561': {'data': bfh('123456')}}, - {'padding': {'padding': bfh('0000000000')}} + {'unknown_tag_561': {'data': bfh(BOB_TLVS['unknown_tag_561'])}}, + {'padding': {'padding': bfh(CAROL_TLVS['padding'])}} ] return create_blinded_path(BLINDING_OVERRIDE_SECRET, [BOB_PUBKEY, CAROL_PUBKEY, DAVE_PUBKEY], final_recipient_data, hop_extras=hop_extras) @@ -222,7 +227,7 @@ class TestOnionMessage(ElectrumTestCase): tlv_stream_name='onionmsg_tlv', blind_fields={ 'next_node_id': {'node_id': BOB_PUBKEY}, - 'next_blinding_override': {'blinding': bfh('031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f')}, + 'next_blinding_override': {'blinding': bfh(ALICE_TLVS['next_blinding_override'])}, } ), ] @@ -237,7 +242,7 @@ class TestOnionMessage(ElectrumTestCase): payload = {'encrypted_recipient_data': {'encrypted_recipient_data': x.get('encrypted_recipient_data')}} if i == len(blinded_path_to_dave.get('path')) - 1: # add final recipient payload - payload['message'] = {'text': b'hello'} + payload['message'] = {'text': bfh(test_vectors['onionmessage']['unknown_tag_1'])} hops_data.append( OnionHopsDataSingle( tlv_stream_name='onionmsg_tlv', From 0b86e3912199a2a0b96e17a2c971a2ce57bfbcfa Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Thu, 13 Feb 2025 14:09:37 +0100 Subject: [PATCH 18/22] onion_messages: additional checks and comments; - check on initial_node_id as the type can in theory contain a sciddir. - log allowed_features if present in recipient_data, with an additional comment describing the handling of allowed_features in the future. - document the SHOULD constraint on onion_message payload size --- electrum/onion_message.py | 10 +++++++++- tests/test_onion_message.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index 93991759c..bb9025713 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -206,6 +206,12 @@ def send_onion_message_to( raise introduction_point = blinded_path['first_node_id'] + if len(introduction_point) != 33: + raise Exception('first_node_id not a nodeid but a sciddir, which is not supported') + # Note: blinded_path specifies type sciddir_or_nodeid for first_node_id + # but only nodeid is supported in onion_message context; + # https://github.com/lightning/bolts/blob/master/04-onion-routing.md + # "MUST set first_node_id to N0" hops_data = [] blinded_node_ids = [] @@ -609,7 +615,9 @@ class OnionMessageManager(Logger): # - `encrypted_data_tlv.allowed_features.features` contains an unknown feature bit (even if it is odd). # - the message uses a feature not included in `encrypted_data_tlv.allowed_features.features`. if 'allowed_features' in recipient_data: - pass # TODO + # Note: These checks will be usecase specific (e.g. BOLT12) and probably should be checked + # by consumers of the message. + self.logger.debug(f'allowed_features={recipient_data["allowed_features"].get("features", b"").hex()}') # - if `path_id` is set and corresponds to a path the reader has previously published in a `reply_path`: # - if the onion message is not a reply to that previous onion: diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 4db5fad10..295f55401 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -104,6 +104,10 @@ class TestOnionMessage(ElectrumTestCase): self.assertEqual(packet.to_bytes(), ONION_MESSAGE_PACKET) def test_onion_message_payload_size(self): + # Note: payload size is not _strictly_ limited to (1300+66, 32768+66), but Electrum only generates these sizes + # However, the spec allows for other payload sizes. + # https://github.com/lightning/bolts/blob/master/04-onion-routing.md + # "SHOULD set onion_message_packet len to 1366 or 32834." hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route([DAVE_PUBKEY], SESSION_KEY) def hops_data_for_message(message): From c3c5aaab3d7f0e123ae19621bb213416f99f3927 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Wed, 19 Feb 2025 16:43:08 +0100 Subject: [PATCH 19/22] onion_messages: add tests for forwards, receive unsolicited. --- electrum/onion_message.py | 51 ++++++++-------- tests/test_onion_message.py | 113 ++++++++++++++++++++++++++++-------- 2 files changed, 115 insertions(+), 49 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index bb9025713..ed83e5b92 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -20,6 +20,7 @@ # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. + import asyncio import copy import io @@ -43,7 +44,6 @@ from electrum.lnutil import LnFeatures from electrum.util import OldTaskGroup, log_exceptions -# do not use util.now, because it rounds to integers def now(): return time.time() @@ -411,12 +411,6 @@ def get_blinded_reply_paths( class Timeout(Exception): pass -class OnionMessageRequest(NamedTuple): - future: asyncio.Future - payload: bytes - node_id_or_blinded_path: bytes - - class OnionMessageManager(Logger): """handle state around onion message sends and receives. - one instance per (ln)wallet @@ -437,12 +431,17 @@ class OnionMessageManager(Logger): FORWARD_RETRY_DELAY = 2 FORWARD_MAX_QUEUE = 3 + class Request(NamedTuple): + future: asyncio.Future + payload: dict + node_id_or_blinded_path: bytes + def __init__(self, lnwallet: 'LNWallet'): Logger.__init__(self) self.network = None # type: Optional['Network'] self.taskgroup = None # type: OldTaskGroup self.lnwallet = lnwallet - self.pending = {} # type: dict[bytes, OnionMessageRequest] + self.pending = {} # type: dict[bytes, OnionMessageManager.Request] self.pending_lock = threading.Lock() self.send_queue = asyncio.PriorityQueue() self.forward_queue = asyncio.PriorityQueue() @@ -559,7 +558,7 @@ class OnionMessageManager(Logger): self.logger.debug(f'submit_send {key=} {payload=} {node_id_or_blinded_path=}') - req = OnionMessageRequest( + req = OnionMessageManager.Request( future=asyncio.Future(), payload=payload, node_id_or_blinded_path=node_id_or_blinded_path @@ -608,6 +607,19 @@ class OnionMessageManager(Logger): # TODO: use payload to determine prefix? return b'electrum' + key + def _get_request_for_path_id(self, recipient_data: dict) -> Request: + path_id = recipient_data.get('path_id', {}).get('data') + if not path_id: + return None + if not path_id[:8] == b'electrum': + self.logger.warning('not a reply to our request (unknown path_id prefix)') + return None + key = path_id[8:] + req = self.pending.get(key) + if req is None: + self.logger.warning('not a reply to our request (unknown request)') + return req + def on_onion_message_received(self, recipient_data: dict, payload: dict) -> None: # we are destination, sanity checks # - if `encrypted_data_tlv` contains `allowed_features`: @@ -622,25 +634,16 @@ class OnionMessageManager(Logger): # - if `path_id` is set and corresponds to a path the reader has previously published in a `reply_path`: # - if the onion message is not a reply to that previous onion: # - MUST ignore the onion message - # TODO: store path_id and lookup here - if 'path_id' not in recipient_data: + req = self._get_request_for_path_id(recipient_data) + if req is None: # unsolicited onion_message self.on_onion_message_received_unsolicited(recipient_data, payload) else: - self.on_onion_message_received_reply(recipient_data, payload) + self.on_onion_message_received_reply(req, recipient_data, payload) - def on_onion_message_received_reply(self, recipient_data: dict, payload: dict) -> None: - # check if this reply is associated with a known request - correl_data = recipient_data['path_id'].get('data') - if not correl_data[:8] == b'electrum': - self.logger.warning('not a reply to our request (unknown path_id prefix)') - return - key = correl_data[8:] - req = self.pending.get(key) - if req is None: - self.logger.warning('not a reply to our request (unknown request)') - return - req.future.set_result((recipient_data, payload)) + def on_onion_message_received_reply(self, request: Request, recipient_data: dict, payload: dict) -> None: + assert request is not None, 'Request is mandatory' + request.future.set_result((recipient_data, payload)) def on_onion_message_received_unsolicited(self, recipient_data: dict, payload: dict) -> None: self.logger.debug('unsolicited onion_message received') diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 295f55401..a65a90be8 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -14,8 +14,8 @@ from electrum.lnonion import ( process_onion_packet, get_bolt04_onion_key, encrypt_onionmsg_data_tlv, get_shared_secrets_along_route, new_onion_packet, ONION_MESSAGE_LARGE_SIZE, HOPS_DATA_SIZE, InvalidPayloadSize) -from electrum.crypto import get_ecdh -from electrum.lnutil import LnFeatures +from electrum.crypto import get_ecdh, privkey_to_pubkey +from electrum.lnutil import LnFeatures, Keypair from electrum.onion_message import blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data, \ OnionMessageManager, NoRouteFound, Timeout from electrum.util import bfh, read_json_file, OldTaskGroup, get_asyncio_loop @@ -304,21 +304,21 @@ class TestOnionMessageManager(ElectrumTestCase): def setUp(self): super().setUp() - self.alice = ECPrivkey(privkey_bytes=b'\x41'*32) - self.alice_pub = self.alice.get_public_key_bytes(compressed=True) - self.bob = ECPrivkey(privkey_bytes=b'\x42'*32) - self.bob_pub = self.bob.get_public_key_bytes(compressed=True) - self.carol = ECPrivkey(privkey_bytes=b'\x43'*32) - self.carol_pub = self.carol.get_public_key_bytes(compressed=True) - self.dave = ECPrivkey(privkey_bytes=b'\x44'*32) - self.dave_pub = self.dave.get_public_key_bytes(compressed=True) - self.eve = ECPrivkey(privkey_bytes=b'\x45'*32) - self.eve_pub = self.eve.get_public_key_bytes(compressed=True) + + def keypair(privkey: ECPrivkey): + priv = privkey.get_secret_bytes() + return Keypair(pubkey=privkey_to_pubkey(priv), privkey=priv) + + self.alice = keypair(ECPrivkey(privkey_bytes=b'\x41'*32)) + self.bob = keypair(ECPrivkey(privkey_bytes=b'\x42'*32)) + self.carol = keypair(ECPrivkey(privkey_bytes=b'\x43'*32)) + self.dave = keypair(ECPrivkey(privkey_bytes=b'\x44'*32)) + self.eve = keypair(ECPrivkey(privkey_bytes=b'\x45'*32)) async def run_test1(self, t): t1 = t.submit_send( payload={'message': {'text': 'alice_timeout'.encode('utf-8')}}, - node_id_or_blinded_path=self.alice_pub) + node_id_or_blinded_path=self.alice.pubkey) with self.assertRaises(Timeout): await t1 @@ -326,7 +326,7 @@ class TestOnionMessageManager(ElectrumTestCase): async def run_test2(self, t): t2 = t.submit_send( payload={'message': {'text': 'bob_slow_timeout'.encode('utf-8')}}, - node_id_or_blinded_path=self.bob_pub) + node_id_or_blinded_path=self.bob.pubkey) with self.assertRaises(Timeout): await t2 @@ -334,7 +334,7 @@ class TestOnionMessageManager(ElectrumTestCase): async def run_test3(self, t, rkey): t3 = t.submit_send( payload={'message': {'text': 'carol_with_immediate_reply'.encode('utf-8')}}, - node_id_or_blinded_path=self.carol_pub, + node_id_or_blinded_path=self.carol.pubkey, key=rkey) t3_result = await t3 @@ -343,7 +343,7 @@ class TestOnionMessageManager(ElectrumTestCase): async def run_test4(self, t, rkey): t4 = t.submit_send( payload={'message': {'text': 'dave_with_slow_reply'.encode('utf-8')}}, - node_id_or_blinded_path=self.dave_pub, + node_id_or_blinded_path=self.dave.pubkey, key=rkey) t4_result = await t4 @@ -352,34 +352,34 @@ class TestOnionMessageManager(ElectrumTestCase): async def run_test5(self, t): t5 = t.submit_send( payload={'message': {'text': 'no_peer'.encode('utf-8')}}, - node_id_or_blinded_path=self.eve_pub) + node_id_or_blinded_path=self.eve.pubkey) with self.assertRaises(NoRouteFound): await t5 - async def test_manager(self): + async def test_request_and_reply(self): n = MockNetwork() k = keypair() q1, q2 = asyncio.Queue(), asyncio.Queue() - lnw = MockLNWallet(local_keypair=k, chans=[], tx_queue=q1, name='test', has_anchors=False) + lnw = MockLNWallet(local_keypair=k, chans=[], tx_queue=q1, name='test_request_and_reply', has_anchors=False) def slow(*args, **kwargs): time.sleep(2*TIME_STEP) def withreply(key, *args, **kwargs): - t.on_onion_message_received_reply({'path_id': {'data': b'electrum' + key}}, {}) + t.on_onion_message_received({'path_id': {'data': b'electrum' + key}}, {}) def slowwithreply(key, *args, **kwargs): time.sleep(2*TIME_STEP) - t.on_onion_message_received_reply({'path_id': {'data': b'electrum' + key}}, {}) + t.on_onion_message_received({'path_id': {'data': b'electrum' + key}}, {}) rkey1 = bfh('0102030405060708') rkey2 = bfh('0102030405060709') - lnw.peers[self.alice_pub] = MockPeer(self.alice_pub) - lnw.peers[self.bob_pub] = MockPeer(self.bob_pub, on_send_message=slow) - lnw.peers[self.carol_pub] = MockPeer(self.carol_pub, on_send_message=partial(withreply, rkey1)) - lnw.peers[self.dave_pub] = MockPeer(self.dave_pub, on_send_message=partial(slowwithreply, rkey2)) + lnw.peers[self.alice.pubkey] = MockPeer(self.alice.pubkey) + lnw.peers[self.bob.pubkey] = MockPeer(self.bob.pubkey, on_send_message=slow) + lnw.peers[self.carol.pubkey] = MockPeer(self.carol.pubkey, on_send_message=partial(withreply, rkey1)) + lnw.peers[self.dave.pubkey] = MockPeer(self.dave.pubkey, on_send_message=partial(slowwithreply, rkey2)) t = OnionMessageManager(lnw) t.start_network(network=n) @@ -404,3 +404,66 @@ class TestOnionMessageManager(ElectrumTestCase): self.logger.debug('stopping manager') await t.stop() await lnw.stop() + + async def test_forward(self): + n = MockNetwork() + q1 = asyncio.Queue() + lnw = MockLNWallet(local_keypair=self.alice, chans=[], tx_queue=q1, name='alice', has_anchors=False) + + self.was_sent = False + + def on_send(to: str, *args, **kwargs): + self.assertEqual(to, 'bob') + self.was_sent = True + # validate what's sent to bob + self.assertEqual(bfh(HOPS[1]['E']), kwargs['blinding']) + message_type, payload = decode_msg(bfh(test_vectors['decrypt']['hops'][1]['onion_message'])) + self.assertEqual(message_type, 'onion_message') + self.assertEqual(payload['onion_message_packet'], kwargs['onion_message_packet']) + + lnw.peers[self.bob.pubkey] = MockPeer(self.bob.pubkey, on_send_message=partial(on_send, 'bob')) + lnw.peers[self.carol.pubkey] = MockPeer(self.carol.pubkey, on_send_message=partial(on_send, 'carol')) + t = OnionMessageManager(lnw) + t.start_network(network=n) + + onionmsg = bfh(test_vectors['onionmessage']['onion_message_packet']) + try: + t.on_onion_message({ + 'blinding': bfh(test_vectors['route']['blinding']), + 'len': len(onionmsg), + 'onion_message_packet': onionmsg + }) + finally: + await asyncio.sleep(TIME_STEP) + + self.logger.debug('stopping manager') + await t.stop() + await lnw.stop() + + self.assertTrue(self.was_sent) + + async def test_receive_unsolicited(self): + n = MockNetwork() + q1 = asyncio.Queue() + lnw = MockLNWallet(local_keypair=self.dave, chans=[], tx_queue=q1, name='dave', has_anchors=False) + + t = OnionMessageManager(lnw) + t.start_network(network=n) + + self.received_unsolicited = False + + def my_on_onion_message_received_unsolicited(*args, **kwargs): + self.received_unsolicited = True + + t.on_onion_message_received_unsolicited = my_on_onion_message_received_unsolicited + packet = bfh(test_vectors['decrypt']['hops'][3]['onion_message']) + message_type, payload = decode_msg(packet) + try: + t.on_onion_message(payload) + self.assertTrue(self.received_unsolicited) + finally: + await asyncio.sleep(TIME_STEP) + + self.logger.debug('stopping manager') + await t.stop() + await lnw.stop() From ad6eb73dd310d47dcf2f8a953484679680da3b81 Mon Sep 17 00:00:00 2001 From: Sander van Grieken Date: Wed, 19 Feb 2025 18:06:04 +0100 Subject: [PATCH 20/22] onion_messages: guard onion message forwarding behind config option EXPERIMENTAL_LN_FORWARD_PAYMENTS (default False) --- electrum/onion_message.py | 4 +++- tests/test_onion_message.py | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index ed83e5b92..d57ccbb87 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -758,5 +758,7 @@ class OnionMessageManager(Logger): if processed_onion_packet.are_we_final: self.on_onion_message_received(recipient_data, payload) - else: + elif self.network.config.EXPERIMENTAL_LN_FORWARD_PAYMENTS: self.on_onion_message_forward(recipient_data, processed_onion_packet.next_packet, blinding, shared_secret) + else: + self.logger.info('onion_message dropped') diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index a65a90be8..2ad9b1de7 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -8,6 +8,7 @@ import logging import electrum_ecc as ecc from electrum_ecc import ECPrivkey +from electrum import SimpleConfig from electrum.lnmsg import decode_msg, OnionWireSerializer from electrum.lnonion import ( OnionHopsDataSingle, OnionPacket, @@ -16,15 +17,17 @@ from electrum.lnonion import ( HOPS_DATA_SIZE, InvalidPayloadSize) from electrum.crypto import get_ecdh, privkey_to_pubkey from electrum.lnutil import LnFeatures, Keypair -from electrum.onion_message import blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data, \ +from electrum.onion_message import ( + blinding_privkey, create_blinded_path, encrypt_onionmsg_tlv_hops_data, OnionMessageManager, NoRouteFound, Timeout +) from electrum.util import bfh, read_json_file, OldTaskGroup, get_asyncio_loop from electrum.logging import console_stderr_handler from . import ElectrumTestCase, test_lnpeer from .test_lnpeer import PutIntoOthersQueueTransport, PeerInTests, keypair -TIME_STEP = 0.01 # run tests 100 x faster +TIME_STEP = 0.01 # run tests 100 x faster OnionMessageManager.SLEEP_DELAY *= TIME_STEP OnionMessageManager.REQUEST_REPLY_TIMEOUT *= TIME_STEP OnionMessageManager.REQUEST_REPLY_RETRY_DELAY *= TIME_STEP @@ -263,6 +266,8 @@ class MockNetwork: def __init__(self): self.asyncio_loop = get_asyncio_loop() self.taskgroup = OldTaskGroup() + self.config = SimpleConfig() + self.config.EXPERIMENTAL_LN_FORWARD_PAYMENTS = True class MockWallet: From 6996574c8e8ef7e712a3b13ce664ae96d6a56d7b Mon Sep 17 00:00:00 2001 From: accumulator Date: Thu, 20 Feb 2025 17:17:14 +0100 Subject: [PATCH 21/22] Update electrum/onion_message.py _get_request_for_path_id return type Co-authored-by: ghost43 --- electrum/onion_message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/electrum/onion_message.py b/electrum/onion_message.py index d57ccbb87..faccefecc 100644 --- a/electrum/onion_message.py +++ b/electrum/onion_message.py @@ -607,7 +607,7 @@ class OnionMessageManager(Logger): # TODO: use payload to determine prefix? return b'electrum' + key - def _get_request_for_path_id(self, recipient_data: dict) -> Request: + def _get_request_for_path_id(self, recipient_data: dict) -> Optional[Request]: path_id = recipient_data.get('path_id', {}).get('data') if not path_id: return None From 7eec7e6a208661afec457d3865d4ea43c266ab17 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 20 Feb 2025 18:05:21 +0100 Subject: [PATCH 22/22] increase sleep time in TestOnionMessageManager.test_forward Co-authored-by: ghost43 --- tests/test_onion_message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_onion_message.py b/tests/test_onion_message.py index 2ad9b1de7..3640eaeac 100644 --- a/tests/test_onion_message.py +++ b/tests/test_onion_message.py @@ -439,7 +439,7 @@ class TestOnionMessageManager(ElectrumTestCase): 'onion_message_packet': onionmsg }) finally: - await asyncio.sleep(TIME_STEP) + await asyncio.sleep(2*TIME_STEP) self.logger.debug('stopping manager') await t.stop()