Resubscribe concurrently and in the background on reconnect (B-31)
_run_once awaited _subscribe_all_users() inline, before starting the header/scripthash consumer tasks, and that method subscribed one user at a time. At thousands of users that's thousands of sequential round-trips during which nothing else ran: tip_height was frozen and an in-flight draw's _wait_for_next_block made zero progress for the entire resubscribe - a reconnect (which the listener already treats as routine, not exceptional) could stall the lottery for minutes. _subscribe_all_users now fans out with bounded concurrency (asyncio.Semaphore, 20 at a time) instead of a serial loop, and one user's failure no longer stops the rest. _run_once now starts it as its own background task, created after the consumer tasks rather than awaited before them, so tip updates and already-subscribed users' notifications keep flowing throughout - its own completion is deliberately not raced against the session-ending tasks (unlike them, it's expected to finish normally), and its failure is logged the same way address_for_new_user's background task is (B-30). Left out: decoupling the listunspent refresh from the subscribe call itself (the third part of the proposed fix) - the periodic DepositReconciler (B-30) already provides a backstop for a slow or delayed initial refresh, so the added complexity wasn't worth it here. Suite grows from 182 to 185 tests, including an end-to-end test against _run_once proving a new tip is processed while a slow resubscribe is still in flight. BUGS.md moves B-31 to Previously fixed.
This commit is contained in:
@@ -35,6 +35,11 @@ _PING_INTERVAL_SECONDS = 60
|
||||
# servers at once — a single slow fallback shouldn't hold up the others.
|
||||
_CORROBORATION_TIMEOUT_SECONDS = 10
|
||||
|
||||
# How many users to resubscribe at once on reconnect (B-31), instead of one at a
|
||||
# time. Bounded rather than unlimited so a huge user base doesn't open thousands
|
||||
# of simultaneous in-flight requests against the one active connection.
|
||||
_RESUBSCRIBE_CONCURRENCY = 20
|
||||
|
||||
|
||||
class ElectrumListener:
|
||||
"""Long-lived background task: keeps one Electrum connection open, subscribes
|
||||
@@ -161,8 +166,6 @@ class ElectrumListener:
|
||||
header = await client.subscribe_headers()
|
||||
self._apply_header(header)
|
||||
|
||||
await self._subscribe_all_users()
|
||||
|
||||
headers_queue = client.notifications("blockchain.headers.subscribe")
|
||||
scripthash_queue = client.notifications("blockchain.scripthash.subscribe")
|
||||
# The consumers below block on their queues forever by design, so they
|
||||
@@ -175,9 +178,25 @@ class ElectrumListener:
|
||||
asyncio.create_task(self._keepalive(client)),
|
||||
asyncio.create_task(client.wait_closed()),
|
||||
]
|
||||
# B-31: resubscribing every user is O(users) sequential round-trips —
|
||||
# at thousands of users that's minutes during which, previously,
|
||||
# nothing above had started yet: tip_height was frozen and an
|
||||
# in-flight draw's _wait_for_next_block made zero progress for the
|
||||
# entire resubscribe. Running it as its own background task instead
|
||||
# of awaiting it inline here means tip updates (and notifications for
|
||||
# whichever users are already subscribed) keep flowing throughout.
|
||||
# It's deliberately not one of the raced `tasks` above: unlike those,
|
||||
# it's expected to finish normally, and its own completion must not
|
||||
# look like the session ending. Any failure partway through is
|
||||
# logged the same way address_for_new_user's background task is
|
||||
# (B-30), and it's cancelled below along with everything else once
|
||||
# the session actually does end.
|
||||
subscribe_task = asyncio.create_task(self._subscribe_all_users())
|
||||
subscribe_task.add_done_callback(self._log_subscribe_all_users_failure)
|
||||
try:
|
||||
done, still_running = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
|
||||
finally:
|
||||
subscribe_task.cancel()
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
for task in done:
|
||||
@@ -189,18 +208,40 @@ class ElectrumListener:
|
||||
await client.close()
|
||||
return True
|
||||
|
||||
def _log_subscribe_all_users_failure(self, task: asyncio.Task) -> None:
|
||||
if task.cancelled():
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc is not None:
|
||||
logger.warning("resubscribing all users failed partway through: %r", exc)
|
||||
|
||||
async def _keepalive(self, client: ElectrumClient) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(_PING_INTERVAL_SECONDS)
|
||||
await client.ping() # raises (and so ends the session) on timeout or a dead socket
|
||||
|
||||
async def _subscribe_all_users(self) -> None:
|
||||
"""B-31: subscribes with bounded concurrency (_RESUBSCRIBE_CONCURRENCY at
|
||||
a time) instead of one user at a time — at thousands of users a serial
|
||||
loop meant thousands of sequential round-trips. One user's failure (a
|
||||
single slow or briefly-erroring request) must not stop the rest from
|
||||
being subscribed, mirroring the same per-item isolation used elsewhere
|
||||
(e.g. tx/confirmation.py's poll_once, deposits/reconcile.py's sweep)."""
|
||||
async with self._session_factory() as session:
|
||||
users = (await session.scalars(select(User))).all()
|
||||
for user in users:
|
||||
|
||||
semaphore = asyncio.Semaphore(_RESUBSCRIBE_CONCURRENCY)
|
||||
|
||||
async def _subscribe_one(user: User) -> None:
|
||||
scripthash = address_to_scripthash(user.address)
|
||||
self._scripthash_to_user[scripthash] = user.id
|
||||
await self._subscribe_and_refresh(scripthash, user.id)
|
||||
async with semaphore:
|
||||
try:
|
||||
await self._subscribe_and_refresh(scripthash, user.id)
|
||||
except Exception:
|
||||
logger.exception("failed to resubscribe user_id=%s", user.id)
|
||||
|
||||
await asyncio.gather(*(_subscribe_one(user) for user in users))
|
||||
|
||||
async def _subscribe_and_refresh(self, scripthash: str, user_id: int) -> None:
|
||||
assert self.client is not None
|
||||
|
||||
Reference in New Issue
Block a user