Merge pull request #10546 from f321x/bolt12_preparation_1

bolt12: preparation 1

electrum/lnonion.py

@@ -193,6 +193,17 @@ def get_blinded_node_id(node_id: bytes, shared_secret: bytes):
     return blinded_node_id.get_public_key_bytes()
 
 
+def blinding_privkey(privkey: bytes, blinding: bytes) -> bytes:
+    shared_secret = get_ecdh(privkey, blinding)
+    b_hmac = get_bolt04_onion_key(b'blinded_node_id', shared_secret)
+    b_hmac_int = int.from_bytes(b_hmac, byteorder="big")
+
+    our_privkey_int = int.from_bytes(privkey, byteorder="big")
+    our_privkey_int = our_privkey_int * b_hmac_int % ecc.CURVE_ORDER
+    our_privkey = our_privkey_int.to_bytes(32, byteorder="big")
+    return our_privkey
+
+
 def new_onion_packet(
     payment_path_pubkeys: Sequence[bytes],
     session_key: bytes,

electrum/onion_message.py

@@ -31,18 +31,19 @@ import dataclasses
 from random import random
 from types import MappingProxyType
 
-from typing import TYPE_CHECKING, Optional, Sequence, NamedTuple, List
+from typing import TYPE_CHECKING, Optional, Sequence, NamedTuple, Tuple, Union
 
 import electrum_ecc as ecc
 
+from electrum.channel_db import get_mychannel_policy
 from electrum.lnrouter import PathEdge
 from electrum.logging import get_logger, Logger
 from electrum.crypto import sha256, get_ecdh
 from electrum.lnmsg import OnionWireSerializer
-from electrum.lnonion import (get_bolt04_onion_key, OnionPacket, process_onion_packet,
+from electrum.lnonion import (get_bolt04_onion_key, OnionPacket, process_onion_packet, blinding_privkey,
                               OnionHopsDataSingle, decrypt_onionmsg_data_tlv, encrypt_onionmsg_data_tlv,
                               get_shared_secrets_along_route, new_onion_packet, encrypt_hops_recipient_data)
-from electrum.lnutil import LnFeatures
+from electrum.lnutil import LnFeatures, MIN_FINAL_CLTV_DELTA_ACCEPTED, MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED
 from electrum.util import OldTaskGroup, log_exceptions
 
 
@@ -55,12 +56,14 @@ if TYPE_CHECKING:
     from electrum.network import Network
     from electrum.lnrouter import NodeInfo
     from electrum.lntransport import LNPeerAddr
+    from electrum.lnchannel import Channel
     from asyncio import Task
 
 logger = get_logger(__name__)
 
 
 REQUEST_REPLY_PATHS_MAX = 3
+PAYMENT_PATHS_MAX = 3
 
 
 class NoRouteFound(Exception):
@@ -75,7 +78,8 @@ def create_blinded_path(
         final_recipient_data: dict,
         *,
         hop_extras: Optional[Sequence[dict]] = None,
-        dummy_hops: Optional[int] = 0
+        dummy_hops: Optional[int] = 0,
+        channels: Optional[Sequence['Channel']] = None,
 ) -> dict:
     # dummy hops could be inserted anywhere in the path, but for compatibility just add them at the end
     # because blinded paths are usually constructed towards ourselves, and we know we can handle dummy hops.
@@ -93,10 +97,19 @@ def create_blinded_path(
         is_non_final_node = i < len(path) - 1
 
         if is_non_final_node:
-            recipient_data = {
-                # TODO: SHOULD add padding data to ensure all encrypted_data_tlv(i) have the same length
-                'next_node_id': {'node_id': path[i+1]}
-            }
+            # spec: alt: short_channel_id instead of next_node_id
+            if channels:  # use short_channel_id for payments
+                scid = channels[i].get_remote_scid_alias() or channels[i].short_channel_id
+                recipient_data = {
+                    # TODO: SHOULD add padding data to ensure all encrypted_data_tlv(i) have the same length
+                    'short_channel_id': {'short_channel_id': scid}
+                }
+            else:
+                recipient_data = {
+                    # TODO: SHOULD add padding data to ensure all encrypted_data_tlv(i) have the same length
+                    'next_node_id': {'node_id': path[i+1]}
+                }
+
             if hop_extras and i < len(hop_extras):  # extra hop data for debugging for now
                 recipient_data.update(hop_extras[i])
         else:
@@ -122,16 +135,14 @@ def create_blinded_path(
     return blinded_path
 
 
-def blinding_privkey(privkey: bytes, blinding: bytes) -> bytes:
-    shared_secret = get_ecdh(privkey, blinding)
-    b_hmac = get_bolt04_onion_key(b'blinded_node_id', shared_secret)
-    b_hmac_int = int.from_bytes(b_hmac, byteorder="big")
-
-    our_privkey_int = int.from_bytes(privkey, byteorder="big")
-    our_privkey_int = our_privkey_int * b_hmac_int % ecc.CURVE_ORDER
-    our_privkey = our_privkey_int.to_bytes(32, byteorder="big")
-
-    return our_privkey
+def encode_blinded_path(blinded_path: dict):
+    with io.BytesIO() as blinded_path_fd:
+        OnionWireSerializer.write_field(
+            fd=blinded_path_fd,
+            field_type='blinded_path',
+            count=1,
+            value=blinded_path)
+        return blinded_path_fd.getvalue()
 
 
 def is_onion_message_node(node_id: bytes, node_info: Optional['NodeInfo']) -> bool:
@@ -159,7 +170,11 @@ def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> Seque
             invoice_amount_msat=10000,  # TODO: do this without amount constraints
             node_filter=lambda x, y: True if x == lnwallet.node_keypair.pubkey else is_onion_message_node(x, y),
             my_sending_channels=my_sending_channels
-        ): return path
+        ):
+            # first edge must be to our peer
+            peer = lnwallet.lnpeermgr.get_peer_by_pubkey(path[0].end_node)
+            assert peer, 'first hop not a peer'
+            return path
 
     # alt: dest is existing peer?
     if lnwallet.lnpeermgr.get_peer_by_pubkey(node_id):
@@ -173,6 +188,74 @@ def create_onion_message_route_to(lnwallet: 'LNWallet', node_id: bytes) -> Seque
     raise NoRouteFound('no path found')
 
 
+def create_route_to_introduction_point(
+    lnwallet: 'LNWallet',
+    blinded_path: dict,
+    introduction_point: bytes,
+    session_key: bytes
+):
+    hops_data = []
+    blinded_node_ids = []
+
+    peer = lnwallet.lnpeermgr.get_peer_by_pubkey(introduction_point)
+    # if blinded path introduction point is our direct peer, no need to route-find
+    if peer:
+        # start of blinded path is our peer
+        path_key = blinded_path['first_path_key']
+        return peer, path_key, hops_data, blinded_node_ids
+
+    path = create_onion_message_route_to(lnwallet, introduction_point)
+
+    # last edge is to introduction point and start of blinded path. remove from route
+    assert path[-1].end_node == introduction_point, 'last hop in route must be introduction point'
+    assert len(path) > 1, "if we are directly connected to the IP, why didn't we return the peer above?"
+
+    peer = lnwallet.lnpeermgr.get_peer_by_pubkey(path[0].end_node)
+    assert peer, "first hop is not a peer"
+
+    # rm last hop (ip-1 -> ip) as it is added explicitly from the blinded path we got (final_hop_pre_ip)
+    path = path[:-1]
+
+    # we construct a route to the introduction point
+    payment_path_pubkeys = [edge.end_node for edge in path]
+    hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route(
+        payment_path_pubkeys,
+        session_key)
+
+    # exclude first hop (us to first node on path): we don't need to a layer for ourselves
+    for edge in path[1:]:
+        hop = OnionHopsDataSingle(
+            tlv_stream_name='onionmsg_tlv',
+            blind_fields={'next_node_id': {'node_id': edge.end_node}},
+        )
+        hops_data.append(hop)
+
+    # final hop pre-ip, add next_path_key_override
+    final_hop_pre_ip = OnionHopsDataSingle(
+        tlv_stream_name='onionmsg_tlv',
+        blind_fields={
+            'next_node_id': {'node_id': introduction_point},
+            'next_path_key_override': {'path_key': blinded_path['first_path_key']},
+        },
+    )
+    hops_data.append(final_hop_pre_ip)
+
+    # encrypt encrypted_data_tlv here
+    for i, hop in enumerate(hops_data):
+        encrypted_recipient_data = encrypt_onionmsg_data_tlv(
+            shared_secret=hop_shared_secrets[i],
+            **hop.blind_fields)
+        payload = dict(hop.payload)
+        payload['encrypted_recipient_data'] = {
+            'encrypted_recipient_data': encrypted_recipient_data
+        }
+        hops_data[i] = dataclasses.replace(hop, payload=payload)
+
+    path_key = ecc.ECPrivkey(session_key).get_public_key_bytes()
+
+    return peer, path_key, hops_data, blinded_node_ids
+
+
 def send_onion_message_to(
         lnwallet: 'LNWallet',
         node_id_or_blinded_path: bytes,
@@ -202,10 +285,10 @@ def send_onion_message_to(
                 # https://github.com/lightning/bolts/blob/master/04-onion-routing.md
                 # "MUST set first_node_id to N0"
 
-            hops_data = []
-            blinded_node_ids = []
-
             if lnwallet.node_keypair.pubkey == introduction_point:
+                hops_data = []
+                blinded_node_ids = []
+
                 # blinded path introduction point is me
                 our_blinding = blinded_path['first_path_key']
                 our_payload = blinded_path['path'][0]
@@ -237,65 +320,13 @@ def send_onion_message_to(
 
             else:
                 # we need a route to introduction point
+                r = create_route_to_introduction_point(lnwallet, blinded_path, introduction_point, session_key)
+                peer, path_key, hops_data, blinded_node_ids = r
+
                 remaining_blinded_path = blinded_path['path']
                 if not isinstance(remaining_blinded_path, list):  # doesn't return list when num items == 1
                     remaining_blinded_path = [remaining_blinded_path]
 
-                peer = lnwallet.lnpeermgr.get_peer_by_pubkey(introduction_point)
-                # if blinded path introduction point is our direct peer, no need to route-find
-                if peer:
-                    # start of blinded path is our peer
-                    path_key = blinded_path['first_path_key']
-                else:
-                    path = create_onion_message_route_to(lnwallet, introduction_point)
-
-                    # first edge must be to our peer
-                    peer = lnwallet.lnpeermgr.get_peer_by_pubkey(path[0].end_node)
-                    assert peer, 'first hop not a peer'
-
-                    # last edge is to introduction point and start of blinded path. remove from route
-                    assert path[-1].end_node == introduction_point, 'last hop in route must be introduction point'
-
-                    path = path[:-1]
-
-                    if len(path) == 0:
-                        path_key = blinded_path['first_path_key']
-                    else:
-                        payment_path_pubkeys = [edge.end_node for edge in path]
-                        hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route(
-                            payment_path_pubkeys,
-                            session_key)
-
-                        hops_data = [
-                            OnionHopsDataSingle(
-                                tlv_stream_name='onionmsg_tlv',
-                                blind_fields={'next_node_id': {'node_id': x.end_node}},
-                            ) for x in path[:-1]
-                        ]
-
-                        # final hop pre-ip, add next_path_key_override
-                        final_hop_pre_ip = OnionHopsDataSingle(
-                            tlv_stream_name='onionmsg_tlv',
-                            blind_fields={
-                                'next_node_id': {'node_id': introduction_point},
-                                'next_path_key_override': {'path_key': blinded_path['first_path_key']},
-                            },
-                        )
-                        hops_data.append(final_hop_pre_ip)
-
-                        # encrypt encrypted_data_tlv here
-                        for i in range(len(hops_data)):
-                            encrypted_recipient_data = encrypt_onionmsg_data_tlv(
-                                shared_secret=hop_shared_secrets[i],
-                                **hops_data[i].blind_fields)
-                            payload = dict(hops_data[i].payload)
-                            payload['encrypted_recipient_data'] = {
-                                'encrypted_recipient_data': encrypted_recipient_data
-                            }
-                            hops_data[i] = dataclasses.replace(hops_data[i], payload=payload)
-
-                        path_key = ecc.ECPrivkey(session_key).get_public_key_bytes()
-
             # append (remaining) blinded path and payload
             blinded_path_blinded_ids = []
             for i, onionmsg_hop in enumerate(remaining_blinded_path):
@@ -369,37 +400,107 @@ def get_blinded_reply_paths(
         path_id: bytes,
         *,
         max_paths: int = REQUEST_REPLY_PATHS_MAX,
-        preferred_node_id: bytes = None
 ) -> Sequence[dict]:
-    """construct a list of blinded reply_paths.
+    """construct a list of blinded reply-paths for onion message.
+    """
+    mydata = {'path_id': {'data': path_id}}  # same path_id used in every reply path
+    paths, payinfo = get_blinded_paths_to_me(lnwallet, mydata, max_paths=max_paths, onion_message=True)
+    return paths
+
+
+def get_blinded_paths_to_me(
+        lnwallet: 'LNWallet',
+        final_recipient_data: dict,
+        *,
+        max_paths: int = PAYMENT_PATHS_MAX,
+        my_channels: Optional[Sequence['Channel']] = None,
+        onion_message: bool = False
+) -> Tuple[Sequence[dict], Sequence[dict]]:
+    """construct a list of blinded paths.
        current logic:
-       - uses current onion_message capable channel peers if exist
-       - otherwise, uses current onion_message capable peers
-       - prefers preferred_node_id if given
-       - reply_path introduction points are direct peers only (TODO: longer reply paths)"""
+       - uses channels peers if not onion_message
+       - uses current onion_message capable channel peers if exist and if onion_message
+       - otherwise, uses current onion_message capable peers if onion_message
+       - reply_path introduction points are direct peers only (TODO: longer paths)"""
     # TODO: build longer paths and/or add dummy hops to increase privacy
-    my_active_channels = [chan for chan in lnwallet.channels.values() if chan.is_active()]
-    my_onionmsg_channels = [chan for chan in my_active_channels if lnwallet.lnpeermgr.get_peer_by_pubkey(chan.node_id) and
-                            lnwallet.lnpeermgr.get_peer_by_pubkey(chan.node_id).their_features.supports(LnFeatures.OPTION_ONION_MESSAGE_OPT)]
-    my_onionmsg_peers = [peer for peer in lnwallet.lnpeermgr.peers.values() if peer.their_features.supports(LnFeatures.OPTION_ONION_MESSAGE_OPT)]
+    if not my_channels:
+        my_active_channels = [chan for chan in lnwallet.channels.values() if chan.is_active()]
+        my_channels = my_active_channels
+
+    if onion_message:
+        my_channels = [chan for chan in my_channels if lnwallet.lnpeermgr.get_peer_by_pubkey(chan.node_id) and
+                       lnwallet.lnpeermgr.get_peer_by_pubkey(chan.node_id).their_features.supports(LnFeatures.OPTION_ONION_MESSAGE_OPT)]
 
     result = []
+    payinfo = []
     mynodeid = lnwallet.node_keypair.pubkey
-    mydata = {'path_id': {'data': path_id}}  # same path_id used in every reply path
-    if len(my_onionmsg_channels):
-        # randomize list, but prefer preferred_node_id
-        rchans = sorted(my_onionmsg_channels, key=lambda x: random() if x.node_id != preferred_node_id else 0)
-        for chan in rchans[:max_paths]:
-            blinded_path = create_blinded_path(os.urandom(32), [chan.node_id, mynodeid], mydata)
-            result.append(blinded_path)
-    elif len(my_onionmsg_peers):
-        # randomize list, but prefer preferred_node_id
-        rpeers = sorted(my_onionmsg_peers, key=lambda x: random() if x.pubkey != preferred_node_id else 0)
-        for peer in rpeers[:max_paths]:
-            blinded_path = create_blinded_path(os.urandom(32), [peer.pubkey, mynodeid], mydata)
-            result.append(blinded_path)
+    local_height = lnwallet.network.get_local_height()
 
-    return result
+    if len(my_channels):
+        # randomize list
+        rchans = sorted(my_channels, key=lambda x: random())
+        for chan in rchans[:max_paths]:
+            hop_extras = None
+            if not onion_message:  # add hop_extras and payinfo, assumption: len(blinded_path) == 2 (us and peer)
+                # get policy
+                cp = get_mychannel_policy(chan.short_channel_id, chan.node_id, {chan.short_channel_id: chan})
+
+                dest_max_cltv_expiry = local_height + MAXIMUM_REMOTE_TO_SELF_DELAY_ACCEPTED
+
+                # TODO: for longer paths (>2), reverse traverse and calculate max_cltv_expiry at each intermediate hop
+                # and determine the cltv delta sums and fee sums of the hops for the payinfo struct.
+                # current assumption is len(blinded_path) == 2 (us and peer)
+                sum_cltv_expiry_delta = cp.cltv_delta
+                sum_fee_base_msat = cp.fee_base_msat
+                sum_fee_proportional_millionths = cp.fee_proportional_millionths
+                # path htlc limits
+                blinded_path_min_htlc_msat = cp.htlc_minimum_msat
+                blinded_path_max_htlc_msat = cp.htlc_maximum_msat
+
+                hop_extras = [{
+                    # spec: MUST include encrypted_data_tlv.payment_relay for each non-final node.
+                    'payment_relay': {
+                        'cltv_expiry_delta': cp.cltv_delta,
+                        'fee_base_msat': cp.fee_base_msat,
+                        'fee_proportional_millionths': cp.fee_proportional_millionths,
+                    },
+                    # spec: MUST set encrypted_data_tlv.payment_constraints for each non-final node and MAY set it for the final node:
+                    #
+                    #     max_cltv_expiry to the largest block height at which the route is allowed to be used, starting
+                    #     from the final node's chosen max_cltv_expiry height at which the route should expire, adding
+                    #     the final node's min_final_cltv_expiry_delta and then adding
+                    #     encrypted_data_tlv.payment_relay.cltv_expiry_delta at each hop.
+                    #
+                    #     htlc_minimum_msat to the largest minimum HTLC value the nodes will allow.
+                    'payment_constraints': {
+                        'max_cltv_expiry': dest_max_cltv_expiry + cp.cltv_delta,
+                        'htlc_minimum_msat': blinded_path_min_htlc_msat
+                    }
+                }]
+                payinfo.append({
+                    'fee_base_msat': sum_fee_base_msat,
+                    'fee_proportional_millionths': sum_fee_proportional_millionths,
+                    'cltv_expiry_delta': sum_cltv_expiry_delta + MIN_FINAL_CLTV_DELTA_ACCEPTED,
+                    'htlc_minimum_msat': blinded_path_min_htlc_msat,
+                    'htlc_maximum_msat': blinded_path_max_htlc_msat,
+                    'flen': 0,
+                    'features': bytes(0)
+                })
+            blinded_path = create_blinded_path(os.urandom(32), [chan.node_id, mynodeid], final_recipient_data,
+                                               hop_extras=hop_extras, channels=[chan] if not onion_message else None)
+            result.append(blinded_path)
+    elif onion_message:
+        # we can use peers even without channels for onion messages
+        my_onionmsg_peers = [peer for peer in lnwallet.lnpeermgr.peers.values() if
+                             peer.their_features.supports(LnFeatures.OPTION_ONION_MESSAGE_OPT)]
+        if len(my_onionmsg_peers):
+            # randomize list
+            rpeers = sorted(my_onionmsg_peers, key=lambda x: random())
+            for peer in rpeers[:max_paths]:
+                blinded_path = create_blinded_path(os.urandom(32), [peer.pubkey, mynodeid], final_recipient_data)
+                result.append(blinded_path)
+
+    return result, payinfo
 
 
 class Timeout(Exception): pass
@@ -415,8 +516,7 @@ class OnionMessageManager(Logger):
     - forwards are best-effort. They should not need retrying, but a queue is used to limit the pacing of forwarding,
       and limiting the number of outstanding forwards. Any onion message forwards arriving when the forward queue
       is full will be dropped.
-
-    TODO: iterate through routes for each request"""
+    """
 
     SLEEP_DELAY = 1
     REQUEST_REPLY_TIMEOUT = 30
@@ -425,10 +525,12 @@ class OnionMessageManager(Logger):
     FORWARD_RETRY_DELAY = 2
     FORWARD_MAX_QUEUE = 3
 
-    class Request(NamedTuple):
-        future: asyncio.Future
-        payload: dict
-        node_id_or_blinded_path: bytes
+    class Request:
+        def __init__(self, *, payload: dict, node_id_or_blinded_paths: Union[bytes, Sequence[bytes]]):
+            self.future = asyncio.Future()
+            self.payload = payload
+            self.node_id_or_blinded_paths = node_id_or_blinded_paths
+            self.current_index: int = 0
 
     def __init__(self, lnwallet: 'LNWallet'):
         Logger.__init__(self)
@@ -525,7 +627,7 @@ class OnionMessageManager(Logger):
             try:
                 self._send_pending_message(key)
             except BaseException as e:
-                self.logger.debug(f'error while sending {key=} {e!r}')
+                self.logger.debug(f'error while sending {key=}: ', exc_info=True)
                 req.future.set_exception(copy.copy(e))
                 # NOTE: above, when passing the caught exception instance e directly it leads to GeneratorExit() in
                 if isinstance(e, NoRouteFound) and e.peer_address:
@@ -542,8 +644,8 @@ class OnionMessageManager(Logger):
     def submit_send(
             self, *,
             payload: dict,
-            node_id_or_blinded_path: bytes,
-            key: bytes = None) -> 'Task':
+            node_id_or_blinded_paths: Union[bytes, Sequence[bytes]],
+            key: Optional[bytes] = None) -> 'Task':
         """Add onion message to queue for sending. Queued onion message payloads
            are supplied with a path_id and a reply_path to determine which request
            corresponds with arriving replies.
@@ -555,13 +657,9 @@ class OnionMessageManager(Logger):
             key = os.urandom(8)
         assert type(key) is bytes and len(key) >= 8
 
-        self.logger.debug(f'submit_send {key=} {payload=} {node_id_or_blinded_path=}')
+        self.logger.debug(f'submit_send {key=} {payload=} {node_id_or_blinded_paths=}')
 
-        req = OnionMessageManager.Request(
-            future=asyncio.Future(),
-            payload=payload,
-            node_id_or_blinded_path=node_id_or_blinded_path
-        )
+        req = OnionMessageManager.Request(payload=payload, node_id_or_blinded_paths=node_id_or_blinded_paths)
         with self.pending_lock:
             if key in self.pending:
                 raise Exception(f'{key=} already exists!')
@@ -584,8 +682,15 @@ class OnionMessageManager(Logger):
         """adds reply_path to payload"""
         req = self.pending.get(key)
         payload = req.payload
-        node_id_or_blinded_path = req.node_id_or_blinded_path
-        self.logger.debug(f'send_pending_message {key=} {payload=} {node_id_or_blinded_path=}')
+
+        # get next path (round robin)
+        dests = req.node_id_or_blinded_paths
+        if isinstance(req.node_id_or_blinded_paths, bytes):
+            dests = [req.node_id_or_blinded_paths]
+        dest = dests[req.current_index]
+        req.current_index = (req.current_index + 1) % len(dests)
+
+        self.logger.debug(f'send_pending_message {key=} {payload=} {dest=}')
 
         final_payload = copy.deepcopy(payload)
 
@@ -598,9 +703,10 @@ class OnionMessageManager(Logger):
 
             final_payload['reply_path'] = {'path': reply_paths}
 
-        # TODO: we should try alternate paths when retrying, this is currently not done.
+        # NOTE: we could also try alternate paths to introduction point (the non-blinded part of the route)
+        # when retrying, this is currently not done.
         # (send_onion_message_to decides path, without knowledge of prev attempts)
-        send_onion_message_to(self.lnwallet, node_id_or_blinded_path, final_payload)
+        send_onion_message_to(self.lnwallet, dest, final_payload)
 
     def _path_id_from_payload_and_key(self, payload: dict, key: bytes) -> bytes:
         # TODO: use payload to determine prefix?

electrum/segwit_addr.py

@@ -43,6 +43,9 @@ class DecodedBech32(NamedTuple):
     data: Optional[Sequence[int]]  # 5-bit ints
 
 
+INVALID_BECH32 = DecodedBech32(None, None, None)
+
+
 def bech32_polymod(values):
     """Internal function that computes the Bech32 checksum."""
     generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3]
@@ -85,26 +88,28 @@ def bech32_encode(encoding: Encoding, hrp: str, data: List[int]) -> str:
     return hrp + '1' + ''.join([CHARSET[d] for d in combined])
 
 
-def bech32_decode(bech: str, *, ignore_long_length=False) -> DecodedBech32:
+def bech32_decode(bech: str, *, ignore_long_length=False, with_checksum=True) -> DecodedBech32:
     """Validate a Bech32/Bech32m string, and determine HRP and data."""
     bech_lower = bech.lower()
     if bech_lower != bech and bech.upper() != bech:
-        return DecodedBech32(None, None, None)
+        return INVALID_BECH32
     pos = bech.rfind('1')
     if pos < 1 or pos + 7 > len(bech) or (not ignore_long_length and len(bech) > 90):
-        return DecodedBech32(None, None, None)
+        return INVALID_BECH32
     # check that HRP only consists of sane ASCII chars
     if any(ord(x) < 33 or ord(x) > 126 for x in bech[:pos+1]):
-        return DecodedBech32(None, None, None)
+        return INVALID_BECH32
     bech = bech_lower
     hrp = bech[:pos]
     try:
         data = [CHARSET_INVERSE[x] for x in bech[pos + 1:]]
     except KeyError:
-        return DecodedBech32(None, None, None)
+        return INVALID_BECH32
+    if not with_checksum:
+        return DecodedBech32(encoding=None, hrp=hrp, data=data)
     encoding = bech32_verify_checksum(hrp, data)
     if encoding is None:
-        return DecodedBech32(None, None, None)
+        return INVALID_BECH32
     return DecodedBech32(encoding=encoding, hrp=hrp, data=data[:-6])
 
 

tests/test_bitcoin.py

@@ -660,6 +660,13 @@ class Test_bitcoin(ElectrumTestCase):
         self.assertEqual(DecodedBech32(None, None, None),
                          segwit_addr.bech32_decode('1p2gdwpf'))
 
+        # without checksum
+        bolt12_str = 'lno1pqps7sjqpgtyzm3qv4uxzmtsd3jjqer9wd3hy6tsw35k7msjzfpy7nz5yqcnygrfdej82um5wf5k2uckyypwa3eyt44h6txtxquqh7lz5djge4afgfjn7k4rgrkuag0jsd5xvxg'
+        self.assertEqual(DecodedBech32(None, None, None),
+                         segwit_addr.bech32_decode(bolt12_str, with_checksum=True, ignore_long_length=True))
+        self.assertEqual(DecodedBech32(None, 'lno', [1, 0, 1, 16, 30, 16, 18, 0, 1, 8, 11, 4, 2, 27, 17, 0, 12, 21, 28, 6, 2, 27, 11, 16, 13, 17, 18, 18, 0, 25, 3, 5, 14, 13, 17, 23, 4, 26, 11, 16, 14, 17, 20, 22, 30, 27, 16, 18, 2, 9, 1, 4, 30, 19, 2, 20, 4, 0, 24, 19, 4, 8, 3, 9, 13, 25, 18, 7, 10, 28, 27, 20, 14, 9, 20, 22, 10, 28, 24, 22, 4, 4, 1, 14, 29, 17, 25, 4, 11, 21, 21, 23, 26, 11, 6, 11, 6, 0, 28, 0, 23, 30, 31, 2, 20, 13, 18, 8, 25, 21, 29, 9, 8, 9, 18, 19, 30, 22, 21, 3, 8, 3, 22, 28, 29, 8, 15, 18, 16, 13, 20, 6, 12, 6, 8]),
+                         segwit_addr.bech32_decode(bolt12_str, with_checksum=False, ignore_long_length=True))
+
 
 class Test_xprv_xpub(ElectrumTestCase):
 

tests/test_lnpeer.py

@@ -377,6 +377,7 @@ class SuccessfulTest(Exception): pass
 
 
 def inject_chan_into_gossipdb(*, channel_db: ChannelDB, graph: Graph, node1name: str, node2name: str) -> None:
+    print(f"injecting channel {node1name} -> {node2name} into channel_db")
     chan_ann_raw = graph.channels[(node1name, node2name)].construct_channel_announcement_without_sigs()[0]
     chan_ann_dict = decode_msg(chan_ann_raw)[1]
     channel_db.add_channel_announcements(chan_ann_dict, trusted=True)

tests/test_onion_message.py

@@ -5,7 +5,8 @@ import time
 import dataclasses
 import logging
 from functools import partial
-from types import MappingProxyType
+from unittest.mock import patch
+from aiorpcx import NetAddress
 
 import electrum_ecc as ecc
 from electrum_ecc import ECPrivkey
@@ -15,16 +16,19 @@ from electrum.lnmsg import decode_msg, OnionWireSerializer
 from electrum.lnonion import (
     OnionHopsDataSingle, OnionPacket, process_onion_packet, get_bolt04_onion_key, encrypt_onionmsg_data_tlv,
     get_shared_secrets_along_route, new_onion_packet, ONION_MESSAGE_LARGE_SIZE, HOPS_DATA_SIZE, InvalidPayloadSize,
-    encrypt_hops_recipient_data)
+    encrypt_hops_recipient_data, blinding_privkey, decrypt_onionmsg_data_tlv)
 from electrum.crypto import get_ecdh, privkey_to_pubkey
-from electrum.lnutil import LnFeatures, Keypair
+from electrum.lntransport import LNPeerAddr
+from electrum.lnutil import LnFeatures, Keypair, MIN_FINAL_CLTV_DELTA_ACCEPTED, REMOTE
 from electrum.onion_message import (
-    blinding_privkey, create_blinded_path,OnionMessageManager, NoRouteFound, Timeout
+    create_blinded_path, OnionMessageManager, NoRouteFound, Timeout,
+    create_route_to_introduction_point, get_blinded_paths_to_me
 )
 from electrum.util import bfh, read_json_file, OldTaskGroup, get_asyncio_loop
 from electrum.logging import console_stderr_handler
 
 from . import ElectrumTestCase
+from .test_lnpeer import TestPeer, inject_chan_into_gossipdb
 
 
 TIME_STEP = 0.01  # run tests 100 x faster
@@ -310,11 +314,12 @@ class TestOnionMessageManager(ElectrumTestCase):
         self.carol = keypair(ECPrivkey(privkey_bytes=b'\x43'*32))
         self.dave = keypair(ECPrivkey(privkey_bytes=b'\x44'*32))
         self.eve = keypair(ECPrivkey(privkey_bytes=b'\x45'*32))
+        self.fred = keypair(ECPrivkey(privkey_bytes=b'\x46'*32))
 
     async def run_test1(self, t):
         t1 = t.submit_send(
             payload={'message': {'text': 'alice_timeout'.encode('utf-8')}},
-            node_id_or_blinded_path=self.alice.pubkey)
+            node_id_or_blinded_paths=self.alice.pubkey)
 
         with self.assertRaises(Timeout):
             await t1
@@ -322,7 +327,7 @@ class TestOnionMessageManager(ElectrumTestCase):
     async def run_test2(self, t):
         t2 = t.submit_send(
             payload={'message': {'text': 'bob_slow_timeout'.encode('utf-8')}},
-            node_id_or_blinded_path=self.bob.pubkey)
+            node_id_or_blinded_paths=self.bob.pubkey)
 
         with self.assertRaises(Timeout):
             await t2
@@ -330,7 +335,7 @@ class TestOnionMessageManager(ElectrumTestCase):
     async def run_test3(self, t, rkey):
         t3 = t.submit_send(
             payload={'message': {'text': 'carol_with_immediate_reply'.encode('utf-8')}},
-            node_id_or_blinded_path=self.carol.pubkey,
+            node_id_or_blinded_paths=self.carol.pubkey,
             key=rkey)
 
         t3_result = await t3
@@ -339,7 +344,7 @@ class TestOnionMessageManager(ElectrumTestCase):
     async def run_test4(self, t, rkey):
         t4 = t.submit_send(
             payload={'message': {'text': 'dave_with_slow_reply'.encode('utf-8')}},
-            node_id_or_blinded_path=self.dave.pubkey,
+            node_id_or_blinded_paths=self.dave.pubkey,
             key=rkey)
 
         t4_result = await t4
@@ -348,15 +353,35 @@ class TestOnionMessageManager(ElectrumTestCase):
     async def run_test5(self, t):
         t5 = t.submit_send(
             payload={'message': {'text': 'no_peer'.encode('utf-8')}},
-            node_id_or_blinded_path=self.eve.pubkey)
+            node_id_or_blinded_paths=self.eve.pubkey)
 
-        with self.assertRaises(NoRouteFound):
+        # will not find route to eve, but has eve's address, but we are configured to not direct connect
+        with self.assertRaises(NoRouteFound) as c:
             await t5
+        self.assertEqual(c.exception.peer_address, LNPeerAddr('localhost', 1234, self.eve.pubkey))
+
+    async def run_test6(self, t, rkey):
+        # bob will not reply, fred will
+        t6 = t.submit_send(
+            payload={'message': {'text': 'send_dest_roundrobin'.encode('utf-8')}},
+            node_id_or_blinded_paths=[self.bob.pubkey, self.fred.pubkey],
+            key=rkey
+        )
+
+        t6_result = await t6
+        self.assertEqual(t6_result, ({'path_id': {'data': b'electrum' + rkey}}, {}))
 
     async def test_request_and_reply(self):
         n = MockNetwork()
         lnw = self.create_mock_lnwallet(name='test_request_and_reply', has_anchors=False)
 
+        # mock add_peer for direct connection fallback
+        async def mock__add_peer(host, port, node_id):
+            mock_peer = MockPeer(pubkey=node_id)
+            # lnw.lnpeermgr._peers[node_id] = mock_peer
+            return mock_peer
+        lnw.lnpeermgr._add_peer = mock__add_peer
+
         def slow(*args, **kwargs):
             time.sleep(2*TIME_STEP)
 
@@ -369,11 +394,14 @@ class TestOnionMessageManager(ElectrumTestCase):
 
         rkey1 = bfh('0102030405060708')
         rkey2 = bfh('0102030405060709')
+        rkey3 = bfh('010203040506070a')
 
         lnw.lnpeermgr._peers[self.alice.pubkey] = MockPeer(self.alice.pubkey)
         lnw.lnpeermgr._peers[self.bob.pubkey] = MockPeer(self.bob.pubkey, on_send_message=slow)
         lnw.lnpeermgr._peers[self.carol.pubkey] = MockPeer(self.carol.pubkey, on_send_message=partial(withreply, rkey1))
         lnw.lnpeermgr._peers[self.dave.pubkey] = MockPeer(self.dave.pubkey, on_send_message=partial(slowwithreply, rkey2))
+        lnw.channel_db._addresses[self.eve.pubkey] = {NetAddress('localhost', '1234'): int(time.time())}
+        lnw.lnpeermgr._peers[self.fred.pubkey] = MockPeer(self.fred.pubkey, on_send_message=partial(withreply, rkey3))
         t = OnionMessageManager(lnw)
         t.start_network(network=n)
 
@@ -385,6 +413,7 @@ class TestOnionMessageManager(ElectrumTestCase):
             await self.run_test3(t, rkey1)
             await self.run_test4(t, rkey2)
             await self.run_test5(t)
+            await self.run_test6(t, rkey3)
             self.logger.debug('tests in parallel')
             async with OldTaskGroup() as group:
                 await group.spawn(self.run_test1(t))
@@ -392,6 +421,7 @@ class TestOnionMessageManager(ElectrumTestCase):
                 await group.spawn(self.run_test3(t, rkey1))
                 await group.spawn(self.run_test4(t, rkey2))
                 await group.spawn(self.run_test5(t))
+                await group.spawn(self.run_test6(t, rkey3))
         finally:
             await asyncio.sleep(TIME_STEP)
 
@@ -461,3 +491,107 @@ class TestOnionMessageManager(ElectrumTestCase):
             self.logger.debug('stopping manager')
             await t.stop()
             await lnw.stop()
+
+
+class TestOnionMessageUtils(TestPeer):
+
+    async def test_get_blinded_paths_to_me_payment(self):
+        # A <- B (alice generates blinded path from bob to herself)
+        graph = self.prepare_chans_and_peers_in_graph(self.GRAPH_DEFINITIONS['single_chan'])
+        alice, bob = graph.workers.values()
+
+        # store bobs channel_update in alice
+        alice_chan = graph.channels[('alice', 'bob')]
+        bob_chan = graph.channels[('bob', 'alice')]
+        bob_update_raw = bob_chan.get_outgoing_gossip_channel_update()
+        bob_update = decode_msg(bob_update_raw)[1]
+        bob_update['raw'] = bob_update_raw
+        alice_chan.set_remote_update(bob_update)
+
+        final_recipient_data = {'path_id': {'data': os.urandom(32)}}
+        paths, payinfos = get_blinded_paths_to_me(alice, final_recipient_data, onion_message=False)
+
+        self.assertEqual(len(paths), 1)
+        self.assertEqual(len(payinfos), 1)
+
+        self.assertEqual(payinfos[0], {
+            'fee_base_msat': bob_chan.forwarding_fee_base_msat,
+            'fee_proportional_millionths': bob_chan.forwarding_fee_proportional_millionths,
+            'cltv_expiry_delta': bob_chan.forwarding_cltv_delta + MIN_FINAL_CLTV_DELTA_ACCEPTED,
+            'htlc_minimum_msat': bob_chan.config[REMOTE].htlc_minimum_msat,
+            'htlc_maximum_msat': min(bob_chan.config[REMOTE].max_htlc_value_in_flight_msat, 1000 * bob_chan.constraints.capacity),
+            'flen': 0,
+            'features': bytes(0),
+        })
+
+        blinded_path = paths[0]
+        self.assertEqual(len(blinded_path['path']), 2)
+        self.assertEqual(blinded_path['first_node_id'], bob.node_keypair.pubkey)
+        self.assertEqual(len(blinded_path['first_path_key']), 33)
+        self.assertEqual(blinded_path['num_hops'], len(blinded_path['path']).to_bytes(length=1, byteorder='big'))
+        self.assertIn('blinded_node_id', blinded_path['path'][0])
+        self.assertIn('encrypted_recipient_data', blinded_path['path'][0])
+
+    async def test_create_route_to_introduction_point(self):
+        # A -- B -- C -- D -- E
+        # Alice constructs route to Edward as introduction point to some blinded path
+        line_graph = self.GRAPH_DEFINITIONS['line_graph']
+        graph = self.prepare_chans_and_peers_in_graph(line_graph)
+        alice, bob, carol, dave, edward = graph.workers.values()
+
+        session_key = os.urandom(32)
+        introduction_point = edward.node_keypair.pubkey
+        first_path_key = ecc.ECPrivkey.generate_random_key().get_public_key_bytes()
+        blinded_path = {
+            'first_path_key': first_path_key,
+        }
+        with self.assertRaises(NoRouteFound):
+            create_route_to_introduction_point(alice, blinded_path, introduction_point, session_key)
+
+        for name, definition in line_graph.items():
+            for channel_partner in definition.get('channels', {}):
+                inject_chan_into_gossipdb(
+                    channel_db=alice.channel_db,
+                    graph=graph,
+                    node1name=name,
+                    node2name=channel_partner,
+                )
+
+        # patch is_onion_message_node so we don't have to inject node announcements
+        with patch('electrum.onion_message.is_onion_message_node', return_value=True):
+            r = create_route_to_introduction_point(alice, blinded_path, introduction_point, session_key)
+        peer, path_key, hops_data, blinded_node_ids = r
+        # alice hands the onion over to bob
+        self.assertEqual(peer.pubkey, bob.lnpeermgr.node_keypair.pubkey)
+
+        self.assertEqual(path_key, ecc.ECPrivkey(session_key).get_public_key_bytes())
+        self.assertEqual(len(hops_data), 3)
+        self.assertEqual(len(hops_data), len(blinded_node_ids))
+
+        # bob unwraps the first layer, sees the next peer, next peer should be carol
+        self.assertEqual(hops_data[0].blind_fields['next_node_id']['node_id'], carol.lnpeermgr.node_keypair.pubkey)
+        self.assertEqual(hops_data[1].blind_fields['next_node_id']['node_id'], dave.lnpeermgr.node_keypair.pubkey)
+        self.assertEqual(hops_data[2].blind_fields['next_node_id']['node_id'], edward.lnpeermgr.node_keypair.pubkey)
+        self.assertEqual(hops_data[2].blind_fields['next_path_key_override']['path_key'], first_path_key)
+
+        # verify that the recipient data is encrypted to the correct node
+        hop_shared_secrets, blinded_node_ids = get_shared_secrets_along_route(
+            (bob.node_keypair.pubkey, carol.node_keypair.pubkey, dave.node_keypair.pubkey),
+            session_key)
+        for hop, ss in zip(hops_data, hop_shared_secrets):
+            encrypted_recipient_data = hop.payload['encrypted_recipient_data']['encrypted_recipient_data']
+            decrypt_onionmsg_data_tlv(
+                shared_secret=ss,
+                encrypted_recipient_data=encrypted_recipient_data,
+            )
+
+        # now Bob is IP, Alice is directly connected to IP
+        introduction_point = bob.node_keypair.pubkey
+        r = create_route_to_introduction_point(alice, blinded_path, introduction_point, session_key)
+        peer, path_key, hops_data, blinded_node_ids = r
+        self.assertEqual(path_key, first_path_key)
+        self.assertEqual(len(hops_data), 0)
+        self.assertEqual(len(blinded_node_ids), 0)
+        alice_bob_peer = alice.lnpeermgr.get_peer_by_pubkey(bob.node_keypair.pubkey)
+        self.assertIsNotNone(alice_bob_peer)
+        self.assertEqual(peer, alice_bob_peer)