Merge pull request #7152 from bitromortac/2103-liquidity-hints

Liquidity hints for pathfinding

electrum/address_synchronizer.py

@@ -32,7 +32,7 @@ from aiorpcx import TaskGroup
 
 from . import bitcoin, util
 from .bitcoin import COINBASE_MATURITY
-from .util import profiler, bfh, TxMinedInfo, UnrelatedTransactionException
+from .util import profiler, bfh, TxMinedInfo, UnrelatedTransactionException, with_lock
 from .transaction import Transaction, TxOutput, TxInput, PartialTxInput, TxOutpoint, PartialTransaction
 from .synchronizer import Synchronizer
 from .verifier import SPV
@@ -98,12 +98,6 @@ class AddressSynchronizer(Logger):
 
         self.load_and_cleanup()
 
-    def with_lock(func):
-        def func_wrapper(self: 'AddressSynchronizer', *args, **kwargs):
-            with self.lock:
-                return func(self, *args, **kwargs)
-        return func_wrapper
-
     def with_transaction_lock(func):
         def func_wrapper(self: 'AddressSynchronizer', *args, **kwargs):
             with self.transaction_lock:

electrum/blockchain.py

@@ -29,7 +29,7 @@ from . import util
 from .bitcoin import hash_encode, int_to_hex, rev_hex
 from .crypto import sha256d
 from . import constants
-from .util import bfh, bh2u
+from .util import bfh, bh2u, with_lock
 from .simple_config import SimpleConfig
 from .logging import get_logger, Logger
 
@@ -195,12 +195,6 @@ class Blockchain(Logger):
         self.lock = threading.RLock()
         self.update_size()
 
-    def with_lock(func):
-        def func_wrapper(self, *args, **kwargs):
-            with self.lock:
-                return func(self, *args, **kwargs)
-        return func_wrapper
-
     @property
     def checkpoints(self):
         return constants.net.CHECKPOINTS

electrum/commands.py

@@ -1092,7 +1092,11 @@ class Commands:
 
     @command('n')
     async def clear_ln_blacklist(self):
-        self.network.path_finder.blacklist.clear()
+        self.network.path_finder.liquidity_hints.clear_blacklist()
+
+    @command('n')
+    async def reset_liquidity_hints(self):
+        self.network.path_finder.liquidity_hints.reset_liquidity_hints()
 
     @command('w')
     async def list_invoices(self, wallet: Abstract_Wallet = None):

electrum/lnhtlc.py

@@ -3,7 +3,7 @@ from typing import Optional, Sequence, Tuple, List, Dict, TYPE_CHECKING, Set
 import threading
 
 from .lnutil import SENT, RECEIVED, LOCAL, REMOTE, HTLCOwner, UpdateAddHtlc, Direction, FeeUpdate
-from .util import bh2u, bfh
+from .util import bh2u, bfh, with_lock
 
 if TYPE_CHECKING:
     from .json_db import StoredDict
@@ -50,12 +50,6 @@ class HTLCManager:
 
         self._init_maybe_active_htlc_ids()
 
-    def with_lock(func):
-        def func_wrapper(self, *args, **kwargs):
-            with self.lock:
-                return func(self, *args, **kwargs)
-        return func_wrapper
-
     @with_lock
     def ctn_latest(self, sub: HTLCOwner) -> int:
         """Return the ctn for the latest (newest that has a valid sig) ctx of sub"""

electrum/lnrouter.py

@@ -27,9 +27,11 @@ import queue
 from collections import defaultdict
 from typing import Sequence, List, Tuple, Optional, Dict, NamedTuple, TYPE_CHECKING, Set
 import time
+from threading import RLock
 import attr
+from math import inf
 
-from .util import bh2u, profiler
+from .util import bh2u, profiler, with_lock
 from .logging import Logger
 from .lnutil import (NUM_MAX_EDGES_IN_PAYMENT_PATH, ShortChannelID, LnFeatures,
                      NBLOCK_CLTV_EXPIRY_TOO_FAR_INTO_FUTURE)
@@ -38,6 +40,11 @@ from .channel_db import ChannelDB, Policy, NodeInfo
 if TYPE_CHECKING:
     from .lnchannel import Channel
 
+DEFAULT_PENALTY_BASE_MSAT = 500  # how much base fee we apply for unknown sending capability of a channel
+DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH = 100  # how much relative fee we apply for unknown sending capability of a channel
+BLACKLIST_DURATION = 3600  # how long (in seconds) a channel remains blacklisted
+HINT_DURATION = 3600  # how long (in seconds) a liquidity hint remains valid
+
 
 class NoChannelPolicy(Exception):
     def __init__(self, short_channel_id: bytes):
@@ -161,12 +168,240 @@ def is_fee_sane(fee_msat: int, *, payment_amount_msat: int) -> bool:
     return False
 
 
+class LiquidityHint:
+    """Encodes the amounts that can and cannot be sent over the direction of a
+    channel and whether the channel is blacklisted.
+
+    A LiquidityHint is the value of a dict, which is keyed to node ids and the
+    channel.
+    """
+    def __init__(self):
+        # use "can_send_forward + can_send_backward < cannot_send_forward + cannot_send_backward" as a sanity check?
+        self._can_send_forward = None
+        self._cannot_send_forward = None
+        self._can_send_backward = None
+        self._cannot_send_backward = None
+        self.blacklist_timestamp = 0
+        self.hint_timestamp = 0
+
+    def is_hint_invalid(self) -> bool:
+        now = int(time.time())
+        return now - self.hint_timestamp > HINT_DURATION
+
+    @property
+    def can_send_forward(self):
+        return None if self.is_hint_invalid() else self._can_send_forward
+
+    @can_send_forward.setter
+    def can_send_forward(self, amount):
+        # we don't want to record less significant info
+        # (sendable amount is lower than known sendable amount):
+        if self._can_send_forward and self._can_send_forward > amount:
+            return
+        self._can_send_forward = amount
+        # we make a sanity check that sendable amount is lower than not sendable amount
+        if self._cannot_send_forward and self._can_send_forward > self._cannot_send_forward:
+            self._cannot_send_forward = None
+
+    @property
+    def can_send_backward(self):
+        return None if self.is_hint_invalid() else self._can_send_backward
+
+    @can_send_backward.setter
+    def can_send_backward(self, amount):
+        if self._can_send_backward and self._can_send_backward > amount:
+            return
+        self._can_send_backward = amount
+        if self._cannot_send_backward and self._can_send_backward > self._cannot_send_backward:
+            self._cannot_send_backward = None
+
+    @property
+    def cannot_send_forward(self):
+        return None if self.is_hint_invalid() else self._cannot_send_forward
+
+    @cannot_send_forward.setter
+    def cannot_send_forward(self, amount):
+        # we don't want to record less significant info
+        # (not sendable amount is higher than known not sendable amount):
+        if self._cannot_send_forward and self._cannot_send_forward < amount:
+            return
+        self._cannot_send_forward = amount
+        if self._can_send_forward and self._can_send_forward > self._cannot_send_forward:
+            self._can_send_forward = None
+        # if we can't send over the channel, we should be able to send in the
+        # reverse direction
+        self.can_send_backward = amount
+
+    @property
+    def cannot_send_backward(self):
+        return None if self.is_hint_invalid() else self._cannot_send_backward
+
+    @cannot_send_backward.setter
+    def cannot_send_backward(self, amount):
+        if self._cannot_send_backward and self._cannot_send_backward < amount:
+            return
+        self._cannot_send_backward = amount
+        if self._can_send_backward and self._can_send_backward > self._cannot_send_backward:
+            self._can_send_backward = None
+        self.can_send_forward = amount
+
+    def can_send(self, is_forward_direction: bool):
+        # make info invalid after some time?
+        if is_forward_direction:
+            return self.can_send_forward
+        else:
+            return self.can_send_backward
+
+    def cannot_send(self, is_forward_direction: bool):
+        # make info invalid after some time?
+        if is_forward_direction:
+            return self.cannot_send_forward
+        else:
+            return self.cannot_send_backward
+
+    def update_can_send(self, is_forward_direction: bool, amount: int):
+        self.hint_timestamp = int(time.time())
+        if is_forward_direction:
+            self.can_send_forward = amount
+        else:
+            self.can_send_backward = amount
+
+    def update_cannot_send(self, is_forward_direction: bool, amount: int):
+        self.hint_timestamp = int(time.time())
+        if is_forward_direction:
+            self.cannot_send_forward = amount
+        else:
+            self.cannot_send_backward = amount
+
+    def __repr__(self):
+        is_blacklisted = False if not self.blacklist_timestamp else int(time.time()) - self.blacklist_timestamp < BLACKLIST_DURATION
+        return f"forward: can send: {self._can_send_forward} msat, cannot send: {self._cannot_send_forward} msat, \n" \
+               f"backward: can send: {self._can_send_backward} msat, cannot send: {self._cannot_send_backward} msat, \n" \
+               f"blacklisted: {is_blacklisted}"
+
+
+class LiquidityHintMgr:
+    """Implements liquidity hints for channels in the graph.
+
+    This class can be used to update liquidity information about channels in the
+    graph. Implements a penalty function for edge weighting in the pathfinding
+    algorithm that favors channels which can route payments and penalizes
+    channels that cannot.
+    """
+    # TODO: incorporate in-flight htlcs
+    # TODO: use timestamps for can/not_send to make them None after some time?
+    # TODO: hints based on node pairs only (shadow channels, non-strict forwarding)?
+    def __init__(self):
+        self.lock = RLock()
+        self._liquidity_hints: Dict[ShortChannelID, LiquidityHint] = {}
+
+    @with_lock
+    def get_hint(self, channel_id: ShortChannelID):
+        hint = self._liquidity_hints.get(channel_id)
+        if not hint:
+            hint = LiquidityHint()
+            self._liquidity_hints[channel_id] = hint
+        return hint
+
+    @with_lock
+    def update_can_send(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID, amount: int):
+        hint = self.get_hint(channel_id)
+        hint.update_can_send(node_from < node_to, amount)
+
+    @with_lock
+    def update_cannot_send(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID, amount: int):
+        hint = self.get_hint(channel_id)
+        hint.update_cannot_send(node_from < node_to, amount)
+
+    def penalty(self, node_from: bytes, node_to: bytes, channel_id: ShortChannelID, amount: int) -> float:
+        """Gives a penalty when sending from node1 to node2 over channel_id with an
+        amount in units of millisatoshi.
+
+        The penalty depends on the can_send and cannot_send values that was
+        possibly recorded in previous payment attempts.
+
+        A channel that can send an amount is assigned a penalty of zero, a
+        channel that cannot send an amount is assigned an infinite penalty.
+        If the sending amount lies between can_send and cannot_send, there's
+        uncertainty and we give a default penalty. The default penalty
+        serves the function of giving a positive offset (the Dijkstra
+        algorithm doesn't work with negative weights), from which we can discount
+        from. There is a competition between low-fee channels and channels where
+        we know with some certainty that they can support a payment. The penalty
+        ultimately boils down to: how much more fees do we want to pay for
+        certainty of payment success? This can be tuned via DEFAULT_PENALTY_BASE_MSAT
+        and DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH. A base _and_ relative penalty
+        was chosen such that the penalty will be able to compete with the regular
+        base and relative fees.
+        """
+        # we only evaluate hints here, so use dict get (to not create many hints with self.get_hint)
+        hint = self._liquidity_hints.get(channel_id)
+        if not hint:
+            can_send, cannot_send = None, None
+        else:
+            can_send = hint.can_send(node_from < node_to)
+            cannot_send = hint.cannot_send(node_from < node_to)
+
+        if cannot_send is not None and amount >= cannot_send:
+            return inf
+        if can_send is not None and amount <= can_send:
+            return 0
+        return fee_for_edge_msat(amount, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH)
+
+    @with_lock
+    def add_to_blacklist(self, channel_id: ShortChannelID):
+        hint = self.get_hint(channel_id)
+        now = int(time.time())
+        hint.blacklist_timestamp = now
+
+    @with_lock
+    def get_blacklist(self) -> Set[ShortChannelID]:
+        now = int(time.time())
+        return set(k for k, v in self._liquidity_hints.items() if now - v.blacklist_timestamp < BLACKLIST_DURATION)
+
+    @with_lock
+    def clear_blacklist(self):
+        for k, v in self._liquidity_hints.items():
+            v.blacklist_timestamp = 0
+
+    @with_lock
+    def reset_liquidity_hints(self):
+        for k, v in self._liquidity_hints.items():
+            v.hint_timestamp = 0
+
+    def __repr__(self):
+        string = "liquidity hints:\n"
+        if self._liquidity_hints:
+            for k, v in self._liquidity_hints.items():
+                string += f"{k}: {v}\n"
+        return string
+
 
 class LNPathFinder(Logger):
 
     def __init__(self, channel_db: ChannelDB):
         Logger.__init__(self)
         self.channel_db = channel_db
+        self.liquidity_hints = LiquidityHintMgr()
+
+    def update_liquidity_hints(
+            self,
+            route: LNPaymentRoute,
+            amount_msat: int,
+            failing_channel: ShortChannelID=None
+    ):
+        # go through the route and record successes until the failing channel is reached,
+        # for the failing channel, add a cannot_send liquidity hint
+        # note: actual routable amounts are slightly different than reported here
+        # as fees would need to be added
+        for r in route:
+            if r.short_channel_id != failing_channel:
+                self.logger.info(f"report {r.short_channel_id} to be able to forward {amount_msat} msat")
+                self.liquidity_hints.update_can_send(r.start_node, r.end_node, r.short_channel_id, amount_msat)
+            else:
+                self.logger.info(f"report {r.short_channel_id} to be unable to forward {amount_msat} msat")
+                self.liquidity_hints.update_cannot_send(r.start_node, r.end_node, r.short_channel_id, amount_msat)
+                break
 
     def _edge_cost(
             self,
@@ -221,19 +456,20 @@ class LNPathFinder(Logger):
                 node_info=node_info)
         if not route_edge.is_sane_to_use(payment_amt_msat):
             return float('inf'), 0  # thanks but no thanks
-
         # Distance metric notes:  # TODO constants are ad-hoc
         # ( somewhat based on https://github.com/lightningnetwork/lnd/pull/1358 )
         # - Edges have a base cost. (more edges -> less likely none will fail)
         # - The larger the payment amount, and the longer the CLTV,
         #   the more irritating it is if the HTLC gets stuck.
         # - Paying lower fees is better. :)
-        base_cost = 500  # one more edge ~ paying 500 msat more fees
         if ignore_costs:
-            return base_cost, 0
+            return DEFAULT_PENALTY_BASE_MSAT, 0
         fee_msat = route_edge.fee_for_edge(payment_amt_msat)
         cltv_cost = route_edge.cltv_expiry_delta * payment_amt_msat * 15 / 1_000_000_000
-        overall_cost = base_cost + fee_msat + cltv_cost
+        # the liquidty penalty takes care we favor edges that should be able to forward
+        # the payment and penalize edges that cannot
+        liquidity_penalty = self.liquidity_hints.penalty(start_node, end_node, short_channel_id, payment_amt_msat)
+        overall_cost = fee_msat + cltv_cost + liquidity_penalty
         return overall_cost, fee_msat
 
     def get_distances(
@@ -243,7 +479,6 @@ class LNPathFinder(Logger):
             nodeB: bytes,
             invoice_amount_msat: int,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
-            blacklist: Set[ShortChannelID] = None,
             private_route_edges: Dict[ShortChannelID, RouteEdge] = None,
     ) -> Dict[bytes, PathEdge]:
         # note: we don't lock self.channel_db, so while the path finding runs,
@@ -252,6 +487,7 @@ class LNPathFinder(Logger):
         # run Dijkstra
         # The search is run in the REVERSE direction, from nodeB to nodeA,
         # to properly calculate compound routing fees.
+        blacklist = self.liquidity_hints.get_blacklist()
         distance_from_start = defaultdict(lambda: float('inf'))
         distance_from_start[nodeB] = 0
         prev_node = {}  # type: Dict[bytes, PathEdge]
@@ -316,7 +552,6 @@ class LNPathFinder(Logger):
             nodeB: bytes,
             invoice_amount_msat: int,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
-            blacklist: Set[ShortChannelID] = None,
             private_route_edges: Dict[ShortChannelID, RouteEdge] = None,
     ) -> Optional[LNPaymentPath]:
         """Return a path from nodeA to nodeB."""
@@ -331,7 +566,6 @@ class LNPathFinder(Logger):
             nodeB=nodeB,
             invoice_amount_msat=invoice_amount_msat,
             my_channels=my_channels,
-            blacklist=blacklist,
             private_route_edges=private_route_edges)
 
         if nodeA not in prev_node:
@@ -394,7 +628,6 @@ class LNPathFinder(Logger):
             invoice_amount_msat: int,
             path = None,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
-            blacklist: Set[ShortChannelID] = None,
             private_route_edges: Dict[ShortChannelID, RouteEdge] = None,
     ) -> Optional[LNPaymentRoute]:
         route = None
@@ -404,7 +637,6 @@ class LNPathFinder(Logger):
                 nodeB=nodeB,
                 invoice_amount_msat=invoice_amount_msat,
                 my_channels=my_channels,
-                blacklist=blacklist,
                 private_route_edges=private_route_edges)
         if path:
             route = self.create_route_from_path(

electrum/lnutil.py

@@ -1360,16 +1360,3 @@ class OnionFailureCodeMetaFlag(IntFlag):
     UPDATE   = 0x1000
 
 
-class ChannelBlackList:
-
-    def __init__(self):
-        self.blacklist = dict() # short_chan_id -> timestamp
-
-    def add(self, short_channel_id: ShortChannelID):
-        now = int(time.time())
-        self.blacklist[short_channel_id] = now
-
-    def get_current_list(self) -> Set[ShortChannelID]:
-        BLACKLIST_DURATION = 3600
-        now = int(time.time())
-        return set(k for k, t in self.blacklist.items() if now - t < BLACKLIST_DURATION)

electrum/lnworker.py

@@ -1210,6 +1210,11 @@ class LNWallet(LNWorker):
                 raise Exception(f"amount_inflight={amount_inflight} < 0")
             log.append(htlc_log)
             if htlc_log.success:
+                # TODO: report every route to liquidity hints for mpp
+                # even in the case of success, we report channels of the
+                # route as being able to send the same amount in the future,
+                # as we assume to not know the capacity
+                self.network.path_finder.update_liquidity_hints(htlc_log.route, htlc_log.amount_msat)
                 return
             # htlc failed
             if len(log) >= attempts:
@@ -1237,7 +1242,7 @@ class LNWallet(LNWorker):
                     raise PaymentFailure(failure_msg.code_name())
             else:
                 self.handle_error_code_from_failed_htlc(
-                    route=route, sender_idx=sender_idx, failure_msg=failure_msg)
+                    route=route, sender_idx=sender_idx, failure_msg=failure_msg, amount=htlc_log.amount_msat)
 
     async def pay_to_route(
             self, *,
@@ -1283,7 +1288,8 @@ class LNWallet(LNWorker):
             *,
             route: LNPaymentRoute,
             sender_idx: int,
-            failure_msg: OnionRoutingFailure) -> None:
+            failure_msg: OnionRoutingFailure,
+            amount: int) -> None:
         code, data = failure_msg.code, failure_msg.data
         # TODO can we use lnmsg.OnionWireSerializer here?
         # TODO update onion_wire.csv
@@ -1296,40 +1302,53 @@ class LNWallet(LNWorker):
             OnionFailureCode.EXPIRY_TOO_SOON: 0,
             OnionFailureCode.CHANNEL_DISABLED: 2,
         }
-        blacklist = False
-        update = False
+
+        # determine a fallback channel to blacklist if we don't get the erring
+        # channel via the payload
+        if sender_idx is None:
+            raise PaymentFailure(failure_msg.code_name())
+        try:
+            fallback_channel = route[sender_idx + 1].short_channel_id
+            node_from = route[sender_idx].start_node
+            node_to = route[sender_idx].end_node
+        except IndexError:
+            raise PaymentFailure(f'payment destination reported error: {failure_msg.code_name()}') from None
+
+        # TODO: handle unknown next peer?
+        # handle failure codes that include a channel update
         if code in failure_codes:
             offset = failure_codes[code]
             channel_update_len = int.from_bytes(data[offset:offset+2], byteorder="big")
             channel_update_as_received = data[offset+2: offset+2+channel_update_len]
             payload = self._decode_channel_update_msg(channel_update_as_received)
+
             if payload is None:
-                self.logger.info(f'could not decode channel_update for failed htlc: {channel_update_as_received.hex()}')
-                blacklist = True
+                self.logger.info(f'could not decode channel_update for failed htlc: '
+                                 f'{channel_update_as_received.hex()}')
+                self.network.path_finder.channel_blacklist.add(fallback_channel)
             else:
+                # apply the channel update or get blacklisted
                 blacklist, update = self._handle_chanupd_from_failed_htlc(
                     payload, route=route, sender_idx=sender_idx)
+
+                # we interpret a temporary channel failure as a liquidity issue
+                # in the channel and update our liquidity hints accordingly
+                if code == OnionFailureCode.TEMPORARY_CHANNEL_FAILURE:
+                    self.network.path_finder.update_liquidity_hints(
+                        route,
+                        amount,
+                        failing_channel=ShortChannelID(payload['short_channel_id']))
+                elif blacklist:
+                    self.network.path_finder.liquidity_hints.add_to_blacklist(
+                        payload['short_channel_id'])
+
+                # if we can't decide on some action, we are stuck
+                if not (blacklist or update):
+                    raise PaymentFailure(failure_msg.code_name())
+
+        # for errors that do not include a channel update
         else:
-            blacklist = True
-
-        if blacklist:
-            # blacklist channel after reporter node
-            # TODO this should depend on the error (even more granularity)
-            # also, we need finer blacklisting (directed edges; nodes)
-            if sender_idx is None:
-                raise PaymentFailure(failure_msg.code_name())
-            try:
-                short_chan_id = route[sender_idx + 1].short_channel_id
-            except IndexError:
-                raise PaymentFailure(f'payment destination reported error: {failure_msg.code_name()}') from None
-            # TODO: for MPP we need to save the amount for which
-            # we saw temporary channel failure
-            self.logger.info(f'blacklisting channel {short_chan_id}')
-            self.network.channel_blacklist.add(short_chan_id)
-
-        # we should not continue if we did not blacklist or update anything
-        if not (blacklist or update):
-            raise PaymentFailure(failure_msg.code_name())
+            self.network.path_finder.liquidity_hints.add_to_blacklist(fallback_channel)
 
     def _handle_chanupd_from_failed_htlc(self, payload, *, route, sender_idx) -> Tuple[bool, bool]:
         blacklist = False
@@ -1599,7 +1618,6 @@ class LNWallet(LNWorker):
             chan.short_channel_id: chan for chan in channels
             if chan.short_channel_id is not None
         }
-        blacklist = self.network.channel_blacklist.get_current_list()
         # Collect all private edges from route hints.
         # Note: if some route hints are multiple edges long, and these paths cross each other,
         #       we allow our path finding to cross the paths; i.e. the route hints are not isolated.
@@ -1631,8 +1649,7 @@ class LNWallet(LNWorker):
                         fee_proportional_millionths=fee_proportional_millionths,
                         cltv_expiry_delta=cltv_expiry_delta,
                         node_features=node_info.features if node_info else 0)
-                if route_edge.short_channel_id not in blacklist:
-                    private_route_edges[route_edge.short_channel_id] = route_edge
+                private_route_edges[route_edge.short_channel_id] = route_edge
                 start_node = end_node
         # now find a route, end to end: between us and the recipient
         try:
@@ -1642,7 +1659,6 @@ class LNWallet(LNWorker):
                 invoice_amount_msat=amount_msat,
                 path=full_path,
                 my_channels=scid_to_my_channels,
-                blacklist=blacklist,
                 private_route_edges=private_route_edges)
         except NoChannelPolicy as e:
             raise NoPathFound() from e

electrum/network.py

@@ -62,7 +62,6 @@ from .version import PROTOCOL_VERSION
 from .simple_config import SimpleConfig
 from .i18n import _
 from .logging import get_logger, Logger
-from .lnutil import ChannelBlackList
 
 if TYPE_CHECKING:
     from .channel_db import ChannelDB
@@ -350,7 +349,6 @@ class Network(Logger, NetworkRetryManager[ServerAddr]):
         self._has_ever_managed_to_connect_to_server = False
 
         # lightning network
-        self.channel_blacklist = ChannelBlackList()
         if self.config.get('run_watchtower', False):
             from . import lnwatcher
             self.local_watchtower = lnwatcher.WatchTower(self)

electrum/tests/test_lnpeer.py

@@ -33,7 +33,7 @@ from electrum import lnmsg
 from electrum.logging import console_stderr_handler, Logger
 from electrum.lnworker import PaymentInfo, RECEIVED
 from electrum.lnonion import OnionFailureCode
-from electrum.lnutil import ChannelBlackList, derive_payment_secret_from_payment_preimage
+from electrum.lnutil import derive_payment_secret_from_payment_preimage
 from electrum.lnutil import LOCAL, REMOTE
 from electrum.invoices import PR_PAID, PR_UNPAID
 
@@ -66,7 +66,6 @@ class MockNetwork:
         self.path_finder = LNPathFinder(self.channel_db)
         self.tx_queue = tx_queue
         self._blockchain = MockBlockchain()
-        self.channel_blacklist = ChannelBlackList()
 
     @property
     def callback_lock(self):
@@ -807,7 +806,7 @@ class TestPeer(TestCaseForTestnet):
             run(f())
 
     @needs_test_with_all_chacha20_implementations
-    def test_payment_with_temp_channel_failure(self):
+    def test_payment_with_temp_channel_failure_and_liquidty_hints(self):
         # prepare channels such that a temporary channel failure happens at c->d
         funds_distribution = {
             'ac': (200_000_000, 200_000_000),  # low fees
@@ -815,12 +814,11 @@ class TestPeer(TestCaseForTestnet):
             'ab': (200_000_000, 200_000_000),  # high fees
             'bd': (200_000_000, 200_000_000),  # high fees
         }
-        # the payment happens in three attempts:
-        # 1. along ac->cd due to low fees with temp channel failure:
+        # the payment happens in two attempts:
+        # 1. along a->c->d due to low fees with temp channel failure:
         #   with chanupd: ORPHANED, private channel update
-        # 2. along ac->cd with temp channel failure:
-        #   with chanupd: ORPHANED, private channel update, but already received, channel gets blacklisted
-        # 3. along ab->bd with success
+        #   c->d gets a liquidity hint and gets blocked
+        # 2. along a->b->d with success
         amount_to_pay = 100_000_000
         graph = self.prepare_chans_and_peers_in_square(funds_distribution)
         peers = graph.all_peers()
@@ -828,9 +826,30 @@ class TestPeer(TestCaseForTestnet):
             self.assertEqual(PR_UNPAID, graph.w_d.get_payment_status(lnaddr.paymenthash))
             result, log = await graph.w_a.pay_invoice(pay_req, attempts=3)
             self.assertTrue(result)
+            self.assertEqual(2, len(log))
             self.assertEqual(PR_PAID, graph.w_d.get_payment_status(lnaddr.paymenthash))
             self.assertEqual(OnionFailureCode.TEMPORARY_CHANNEL_FAILURE, log[0].failure_msg.code)
-            self.assertEqual(OnionFailureCode.TEMPORARY_CHANNEL_FAILURE, log[1].failure_msg.code)
+
+            liquidity_hints = graph.w_a.network.path_finder.liquidity_hints
+            pubkey_a = graph.w_a.node_keypair.pubkey
+            pubkey_b = graph.w_b.node_keypair.pubkey
+            pubkey_c = graph.w_c.node_keypair.pubkey
+            pubkey_d = graph.w_d.node_keypair.pubkey
+            # check liquidity hints for failing route:
+            hint_ac = liquidity_hints.get_hint(graph.chan_ac.short_channel_id)
+            hint_cd = liquidity_hints.get_hint(graph.chan_cd.short_channel_id)
+            self.assertEqual(amount_to_pay, hint_ac.can_send(pubkey_a < pubkey_c))
+            self.assertEqual(None, hint_ac.cannot_send(pubkey_a < pubkey_c))
+            self.assertEqual(None, hint_cd.can_send(pubkey_c < pubkey_d))
+            self.assertEqual(amount_to_pay, hint_cd.cannot_send(pubkey_c < pubkey_d))
+            # check liquidity hints for successful route:
+            hint_ab = liquidity_hints.get_hint(graph.chan_ab.short_channel_id)
+            hint_bd = liquidity_hints.get_hint(graph.chan_bd.short_channel_id)
+            self.assertEqual(amount_to_pay, hint_ab.can_send(pubkey_a < pubkey_b))
+            self.assertEqual(None, hint_ab.cannot_send(pubkey_a < pubkey_b))
+            self.assertEqual(amount_to_pay, hint_bd.can_send(pubkey_b < pubkey_d))
+            self.assertEqual(None, hint_bd.cannot_send(pubkey_b < pubkey_d))
+
             raise PaymentDone()
         async def f():
             async with TaskGroup() as group:

electrum/tests/test_lnrouter.py

@@ -1,21 +1,31 @@
+from math import inf
 import unittest
 import tempfile
 import shutil
 import asyncio
 
 from electrum.util import bh2u, bfh, create_and_start_event_loop
+from electrum.lnutil import ShortChannelID
 from electrum.lnonion import (OnionHopsDataSingle, new_onion_packet,
                               process_onion_packet, _decode_onion_error, decode_onion_error,
                               OnionFailureCode, OnionPacket)
 from electrum import bitcoin, lnrouter
 from electrum.constants import BitcoinTestnet
 from electrum.simple_config import SimpleConfig
-from electrum.lnrouter import PathEdge
+from electrum.lnrouter import PathEdge, LiquidityHintMgr, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH, DEFAULT_PENALTY_BASE_MSAT, fee_for_edge_msat
 
 from . import TestCaseForTestnet
 from .test_bitcoin import needs_test_with_all_chacha20_implementations
 
 
+def channel(number: int) -> ShortChannelID:
+    return ShortChannelID(bfh(format(number, '016x')))
+
+
+def node(character: str) -> bytes:
+    return b'\x02' + f'{character}'.encode() * 32
+
+
 class Test_LNRouter(TestCaseForTestnet):
 
     def setUp(self):
@@ -28,7 +38,24 @@ class Test_LNRouter(TestCaseForTestnet):
         self._loop_thread.join(timeout=1)
         super().tearDown()
 
-    def test_find_path_for_payment(self):
+    def prepare_graph(self):
+        """
+        Network topology with channel ids:
+                3
+            A  ---  B
+            |    2/ |
+          6 |   E   | 1
+            | /5 \7 |
+            D  ---  C
+                4
+        valid routes from A -> E:
+        A -3-> B -2-> E
+        A -6-> D -5-> E
+        A -6-> D -4-> C -7-> E
+        A -3-> B -1-> C -7-> E
+        A -6-> D -4-> C -1-> B -2-> E
+        A -3-> B -1-> C -4-> D -5-> E
+        """
         class fake_network:
             config = self.config
             asyncio_loop = asyncio.get_event_loop()
@@ -37,67 +64,193 @@ class Test_LNRouter(TestCaseForTestnet):
             interface = None
         fake_network.channel_db = lnrouter.ChannelDB(fake_network())
         fake_network.channel_db.data_loaded.set()
-        cdb = fake_network.channel_db
-        path_finder = lnrouter.LNPathFinder(cdb)
-        self.assertEqual(cdb.num_channels, 0)
-        cdb.add_channel_announcements({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02cccccccccccccccccccccccccccccccc',
-                                     'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02cccccccccccccccccccccccccccccccc',
-                                     'short_channel_id': bfh('0000000000000001'),
-                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
-                                     'len': 0, 'features': b''}, trusted=True)
-        self.assertEqual(cdb.num_channels, 1)
-        cdb.add_channel_announcements({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
-                                     'bitcoin_key_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
-                                     'short_channel_id': bfh('0000000000000002'),
-                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
-                                     'len': 0, 'features': b''}, trusted=True)
-        cdb.add_channel_announcements({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
-                                     'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
-                                     'short_channel_id': bfh('0000000000000003'),
-                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
-                                     'len': 0, 'features': b''}, trusted=True)
-        cdb.add_channel_announcements({'node_id_1': b'\x02cccccccccccccccccccccccccccccccc', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd',
-                                     'bitcoin_key_1': b'\x02cccccccccccccccccccccccccccccccc', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd',
-                                     'short_channel_id': bfh('0000000000000004'),
-                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
-                                     'len': 0, 'features': b''}, trusted=True)
-        cdb.add_channel_announcements({'node_id_1': b'\x02dddddddddddddddddddddddddddddddd', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
-                                     'bitcoin_key_1': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
-                                     'short_channel_id': bfh('0000000000000005'),
-                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
-                                     'len': 0, 'features': b''}, trusted=True)
-        cdb.add_channel_announcements({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd',
-                                     'bitcoin_key_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd',
-                                     'short_channel_id': bfh('0000000000000006'),
-                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
-                                     'len': 0, 'features': b''}, trusted=True)
+        self.cdb = fake_network.channel_db
+        self.path_finder = lnrouter.LNPathFinder(self.cdb)
+        self.assertEqual(self.cdb.num_channels, 0)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('b'), 'node_id_2': node('c'),
+            'bitcoin_key_1': node('b'), 'bitcoin_key_2': node('c'),
+            'short_channel_id': channel(1),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
+        self.assertEqual(self.cdb.num_channels, 1)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('b'), 'node_id_2': node('e'),
+            'bitcoin_key_1': node('b'), 'bitcoin_key_2': node('e'),
+            'short_channel_id': channel(2),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('a'), 'node_id_2': node('b'),
+            'bitcoin_key_1': node('a'), 'bitcoin_key_2': node('b'),
+            'short_channel_id': channel(3),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('c'), 'node_id_2': node('d'),
+            'bitcoin_key_1': node('c'), 'bitcoin_key_2': node('d'),
+            'short_channel_id': channel(4),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('d'), 'node_id_2': node('e'),
+            'bitcoin_key_1': node('d'), 'bitcoin_key_2': node('e'),
+            'short_channel_id': channel(5),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('a'), 'node_id_2': node('d'),
+            'bitcoin_key_1': node('a'), 'bitcoin_key_2': node('d'),
+            'short_channel_id': channel(6),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
+        self.cdb.add_channel_announcements({
+            'node_id_1': node('c'), 'node_id_2': node('e'),
+            'bitcoin_key_1': node('c'), 'bitcoin_key_2': node('e'),
+            'short_channel_id': channel(7),
+            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
+            'len': 0, 'features': b''
+        }, trusted=True)
         def add_chan_upd(payload):
-            cdb.add_channel_update(payload, verify=False)
-        add_chan_upd({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 99999999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        add_chan_upd({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
-        path = path_finder.find_path_for_payment(
-            nodeA=b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
-            nodeB=b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
-            invoice_amount_msat=100000)
-        self.assertEqual([PathEdge(start_node=b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', end_node=b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', short_channel_id=bfh('0000000000000003')),
-                          PathEdge(start_node=b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', end_node=b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', short_channel_id=bfh('0000000000000002')),
-                         ], path)
-        route = path_finder.create_route_from_path(path)
-        self.assertEqual(b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', route[0].node_id)
-        self.assertEqual(bfh('0000000000000003'),                 route[0].short_channel_id)
+            self.cdb.add_channel_update(payload, verify=False)
+        add_chan_upd({'short_channel_id': channel(1), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(1), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(2), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(2), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(3), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(3), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(4), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(4), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(5), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(5), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(6), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 200, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(6), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(7), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+        add_chan_upd({'short_channel_id': channel(7), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
+
+    def test_find_path_for_payment(self):
+        self.prepare_graph()
+        amount_to_send = 100000
+
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual([
+            PathEdge(start_node=node('a'), end_node=node('b'), short_channel_id=channel(3)),
+            PathEdge(start_node=node('b'), end_node=node('e'), short_channel_id=channel(2)),
+        ], path)
+
+        route = self.path_finder.create_route_from_path(path)
+        self.assertEqual(node('b'), route[0].node_id)
+        self.assertEqual(channel(3), route[0].short_channel_id)
+
+        self.cdb.stop()
+        asyncio.run_coroutine_threadsafe(self.cdb.stopped_event.wait(), self.asyncio_loop).result()
+
+    def test_find_path_liquidity_hints_failure(self):
+        self.prepare_graph()
+        amount_to_send = 100000
+
+        """
+        assume failure over channel 2, B -> E
+        A -3-> B |-2-> E
+        A -6-> D -5-> E  <= chosen path
+        A -6-> D -4-> C -7-> E
+        A -3-> B -1-> C -7-> E
+        A -6-> D -4-> C -1-> B -2-> E
+        A -3-> B -1-> C -4-> D -5-> E
+        """
+        self.path_finder.liquidity_hints.update_cannot_send(node('b'), node('e'), channel(2), amount_to_send - 1)
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual(channel(6), path[0].short_channel_id)
+        self.assertEqual(channel(5), path[1].short_channel_id)
+
+        """
+        assume failure over channel 5, D -> E
+        A -3-> B |-2-> E
+        A -6-> D |-5-> E
+        A -6-> D -4-> C -7-> E
+        A -3-> B -1-> C -7-> E  <= chosen path
+        A -6-> D -4-> C -1-> B |-2-> E
+        A -3-> B -1-> C -4-> D |-5-> E
+        """
+        self.path_finder.liquidity_hints.update_cannot_send(node('d'), node('e'), channel(5), amount_to_send - 1)
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual(channel(3), path[0].short_channel_id)
+        self.assertEqual(channel(1), path[1].short_channel_id)
+        self.assertEqual(channel(7), path[2].short_channel_id)
+
+        """
+        assume success over channel 4, D -> C
+        A -3-> B |-2-> E
+        A -6-> D |-5-> E
+        A -6-> D -4-> C -7-> E  <= chosen path
+        A -3-> B -1-> C -7-> E
+        A -6-> D -4-> C -1-> B |-2-> E
+        A -3-> B -1-> C -4-> D |-5-> E
+        """
+        self.path_finder.liquidity_hints.update_can_send(node('d'), node('c'), channel(4), amount_to_send + 1000)
+        path = self.path_finder.find_path_for_payment(
+            nodeA=node('a'),
+            nodeB=node('e'),
+            invoice_amount_msat=amount_to_send)
+        self.assertEqual(channel(6), path[0].short_channel_id)
+        self.assertEqual(channel(4), path[1].short_channel_id)
+        self.assertEqual(channel(7), path[2].short_channel_id)
+
+        self.cdb.stop()
+        asyncio.run_coroutine_threadsafe(self.cdb.stopped_event.wait(), self.asyncio_loop).result()
+
+    def test_liquidity_hints(self):
+        liquidity_hints = LiquidityHintMgr()
+        node_from = bytes(0)
+        node_to = bytes(1)
+        channel_id = ShortChannelID.from_components(0, 0, 0)
+        amount_to_send = 1_000_000
+
+        # check default penalty
+        self.assertEqual(
+            fee_for_edge_msat(amount_to_send, DEFAULT_PENALTY_BASE_MSAT, DEFAULT_PENALTY_PROPORTIONAL_MILLIONTH),
+            liquidity_hints.penalty(node_from, node_to, channel_id, amount_to_send)
+        )
+        liquidity_hints.update_can_send(node_from, node_to, channel_id, 1_000_000)
+        liquidity_hints.update_cannot_send(node_from, node_to, channel_id, 2_000_000)
+        hint = liquidity_hints.get_hint(channel_id)
+        self.assertEqual(1_000_000, hint.can_send(node_from < node_to))
+        self.assertEqual(None, hint.cannot_send(node_to < node_from))
+        self.assertEqual(2_000_000, hint.cannot_send(node_from < node_to))
+        # the can_send backward hint is set automatically
+        self.assertEqual(2_000_000, hint.can_send(node_to < node_from))
+
+        # check penalties
+        self.assertEqual(0., liquidity_hints.penalty(node_from, node_to, channel_id, 1_000_000))
+        self.assertEqual(650, liquidity_hints.penalty(node_from, node_to, channel_id, 1_500_000))
+        self.assertEqual(inf, liquidity_hints.penalty(node_from, node_to, channel_id, 2_000_000))
+
+        # test that we don't overwrite significant info with less significant info
+        liquidity_hints.update_can_send(node_from, node_to, channel_id, 500_000)
+        hint = liquidity_hints.get_hint(channel_id)
+        self.assertEqual(1_000_000, hint.can_send(node_from < node_to))
+
+        # test case when can_send > cannot_send
+        liquidity_hints.update_can_send(node_from, node_to, channel_id, 3_000_000)
+        hint = liquidity_hints.get_hint(channel_id)
+        self.assertEqual(3_000_000, hint.can_send(node_from < node_to))
+        self.assertEqual(None, hint.cannot_send(node_from < node_to))
 
-        cdb.stop()
-        asyncio.run_coroutine_threadsafe(cdb.stopped_event.wait(), self.asyncio_loop).result()
 
     @needs_test_with_all_chacha20_implementations
     def test_new_onion_packet_legacy(self):

electrum/util.py

@@ -776,7 +776,7 @@ mainnet_block_explorers = {
     'mempool.space': ('https://mempool.space/',
                         {'tx': 'tx/', 'addr': 'address/'}),
     'mempool.emzy.de': ('https://mempool.emzy.de/',
-                        {'tx': 'tx/', 'addr': 'address/'}),  
+                        {'tx': 'tx/', 'addr': 'address/'}),
     'OXT.me': ('https://oxt.me/',
                         {'tx': 'transaction/', 'addr': 'address/'}),
     'smartbit.com.au': ('https://www.smartbit.com.au/',
@@ -797,7 +797,7 @@ testnet_block_explorers = {
     'Blockstream.info': ('https://blockstream.info/testnet/',
                         {'tx': 'tx/', 'addr': 'address/'}),
     'mempool.space': ('https://mempool.space/testnet/',
-                        {'tx': 'tx/', 'addr': 'address/'}),    
+                        {'tx': 'tx/', 'addr': 'address/'}),
     'smartbit.com.au': ('https://testnet.smartbit.com.au/',
                        {'tx': 'tx/', 'addr': 'address/'}),
     'system default': ('blockchain://000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943/',
@@ -1116,6 +1116,14 @@ def ignore_exceptions(func):
     return wrapper
 
 
+def with_lock(func):
+    """Decorator to enforce a lock on a function call."""
+    def func_wrapper(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return func_wrapper
+
+
 class TxMinedInfo(NamedTuple):
     height: int                        # height of block that mined tx
     conf: Optional[int] = None         # number of confirmations, SPV verified (None means unknown)