Resubscribe concurrently and in the background on reconnect (B-31)
_run_once awaited _subscribe_all_users() inline, before starting the header/scripthash consumer tasks, and that method subscribed one user at a time. At thousands of users that's thousands of sequential round-trips during which nothing else ran: tip_height was frozen and an in-flight draw's _wait_for_next_block made zero progress for the entire resubscribe - a reconnect (which the listener already treats as routine, not exceptional) could stall the lottery for minutes. _subscribe_all_users now fans out with bounded concurrency (asyncio.Semaphore, 20 at a time) instead of a serial loop, and one user's failure no longer stops the rest. _run_once now starts it as its own background task, created after the consumer tasks rather than awaited before them, so tip updates and already-subscribed users' notifications keep flowing throughout - its own completion is deliberately not raced against the session-ending tasks (unlike them, it's expected to finish normally), and its failure is logged the same way address_for_new_user's background task is (B-30). Left out: decoupling the listunspent refresh from the subscribe call itself (the third part of the proposed fix) - the periodic DepositReconciler (B-30) already provides a backstop for a slow or delayed initial refresh, so the added complexity wasn't worth it here. Suite grows from 182 to 185 tests, including an end-to-end test against _run_once proving a new tip is processed while a slow resubscribe is still in flight. BUGS.md moves B-31 to Previously fixed.
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
"""Listener-level behaviour: server rotation on failure (the fallback-servers
|
||||
feature), the chain-tip monotonicity guard (B-19), header validation and
|
||||
multi-server corroboration (B-28), and the new-user subscribe task's retention
|
||||
and error logging (B-30).
|
||||
multi-server corroboration (B-28), the new-user subscribe task's retention and
|
||||
error logging (B-30), and bounded-concurrency, non-blocking resubscribe on
|
||||
reconnect (B-31).
|
||||
|
||||
The reconnect loop itself (B-01) is covered from the client side in
|
||||
test_electrum_client.py — what's asserted here is that the listener *acts* on a
|
||||
@@ -442,7 +443,7 @@ async def _seed_funded_user(session_factory, *, username: str, address: str) ->
|
||||
_UNRELATED_ENTRY = [{"tx_hash": "99" * 32, "tx_pos": 0, "height": 100, "value": 1_000_000}]
|
||||
|
||||
|
||||
async def testrefresh_user_marks_a_utxo_spent_once_others_corroborate_it(session_factory):
|
||||
async def test_refresh_user_marks_a_utxo_spent_once_others_corroborate_it(session_factory):
|
||||
user_id = await _seed_funded_user(session_factory, username="bob", address="plm1q0xcqpzrky6eff2g52qdye53xkk9jxkvraxkkwd")
|
||||
|
||||
others_factory = await _listunspent_client_factory(
|
||||
@@ -464,7 +465,7 @@ async def testrefresh_user_marks_a_utxo_spent_once_others_corroborate_it(session
|
||||
assert user.cached_balance_sats == 1_000_000
|
||||
|
||||
|
||||
async def testrefresh_user_does_not_mark_when_corroboration_fails(session_factory):
|
||||
async def test_refresh_user_does_not_mark_when_corroboration_fails(session_factory):
|
||||
"""The single most important case: our own connection alone reporting the
|
||||
UTXO missing must not be enough — before B-29 this zeroed the balance on one
|
||||
bad reply."""
|
||||
@@ -487,3 +488,145 @@ async def testrefresh_user_does_not_mark_when_corroboration_fails(session_factor
|
||||
user = await session.get(User, user_id)
|
||||
# Untouched, plus the unrelated entry credited alongside it.
|
||||
assert user.cached_balance_sats == 21_000_000
|
||||
|
||||
|
||||
# --- B-31: resubscribing on reconnect must be bounded-concurrency and must not
|
||||
# block tip updates (and so an in-flight draw) for its entire duration. -----------
|
||||
|
||||
|
||||
def _fake_address(i: int) -> str:
|
||||
"""A real, decodable PLM bech32 P2WPKH address (address_to_scripthash
|
||||
actually parses it) — distinct per index, since User.address is unique."""
|
||||
from embit import script
|
||||
|
||||
from app.wallet.plm_network import PLM_MAINNET
|
||||
|
||||
payload = (i + 1).to_bytes(20, "big")
|
||||
return script.Script(b"\x00\x14" + payload).address(network=PLM_MAINNET)
|
||||
|
||||
|
||||
async def _seed_users(session_factory, count: int) -> None:
|
||||
async with session_factory() as session:
|
||||
for i in range(count):
|
||||
session.add(
|
||||
User(username=f"user{i}", password_hash="x", derivation_index=i, address=_fake_address(i))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def test_subscribe_all_users_bounds_concurrency(session_factory):
|
||||
"""B-31: at thousands of users, subscribing one at a time meant thousands of
|
||||
sequential round-trips. Concurrency must be bounded (not unlimited either —
|
||||
a huge user base shouldn't open thousands of simultaneous requests)."""
|
||||
user_count = 45
|
||||
await _seed_users(session_factory, user_count)
|
||||
|
||||
listener = ElectrumListener(lambda endpoint: None, session_factory, _ENDPOINTS)
|
||||
in_flight = 0
|
||||
max_in_flight = 0
|
||||
calls = []
|
||||
|
||||
async def fake_subscribe_and_refresh(scripthash, user_id):
|
||||
nonlocal in_flight, max_in_flight
|
||||
in_flight += 1
|
||||
max_in_flight = max(max_in_flight, in_flight)
|
||||
calls.append(user_id)
|
||||
await asyncio.sleep(0) # yield, so genuinely-concurrent calls interleave
|
||||
in_flight -= 1
|
||||
|
||||
listener._subscribe_and_refresh = fake_subscribe_and_refresh
|
||||
|
||||
await listener._subscribe_all_users()
|
||||
|
||||
assert len(calls) == user_count
|
||||
assert 1 < max_in_flight <= 20 # bounded, and actually concurrent (not serial)
|
||||
|
||||
|
||||
async def test_subscribe_all_users_continues_past_a_failing_user(session_factory):
|
||||
await _seed_users(session_factory, 5)
|
||||
listener = ElectrumListener(lambda endpoint: None, session_factory, _ENDPOINTS)
|
||||
succeeded = []
|
||||
|
||||
async def flaky_subscribe_and_refresh(scripthash, user_id):
|
||||
if user_id == 3:
|
||||
raise ConnectionResetError("dropped mid-subscribe")
|
||||
succeeded.append(user_id)
|
||||
|
||||
listener._subscribe_and_refresh = flaky_subscribe_and_refresh
|
||||
|
||||
await listener._subscribe_all_users() # must not raise
|
||||
|
||||
assert succeeded == [1, 2, 4, 5]
|
||||
|
||||
|
||||
class _FakeConnectClient:
|
||||
"""A minimally-real ElectrumClient double: enough of connect/subscribe/notify/
|
||||
ping/wait_closed/close to drive ElectrumListener._run_once end-to-end."""
|
||||
|
||||
def __init__(self, header: dict):
|
||||
self._header = header
|
||||
self._queues: dict[str, asyncio.Queue] = {}
|
||||
self._closed = asyncio.Event()
|
||||
|
||||
async def connect(self):
|
||||
pass
|
||||
|
||||
async def subscribe_headers(self):
|
||||
return self._header
|
||||
|
||||
def notifications(self, method: str) -> asyncio.Queue:
|
||||
return self._queues.setdefault(method, asyncio.Queue())
|
||||
|
||||
async def ping(self):
|
||||
pass
|
||||
|
||||
async def wait_closed(self):
|
||||
await self._closed.wait()
|
||||
|
||||
async def close(self):
|
||||
self._closed.set()
|
||||
|
||||
|
||||
async def _wait_until(predicate, *, timeout: float = 2.0, interval: float = 0.01) -> None:
|
||||
async def _poll():
|
||||
while not predicate():
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
await asyncio.wait_for(_poll(), timeout=timeout)
|
||||
|
||||
|
||||
async def test_run_once_keeps_consuming_headers_while_resubscribing(session_factory):
|
||||
"""The core B-31 fix: before this, _subscribe_all_users ran to completion
|
||||
*before* the header-consuming task even started, so a reconnect with many
|
||||
users froze tip_height — and so _wait_for_next_block's draw wait — for the
|
||||
entire resubscribe. It must now keep advancing while resubscribing is still
|
||||
in flight."""
|
||||
await _seed_users(session_factory, 3)
|
||||
|
||||
header_hex = _mine_header("00" * 32)
|
||||
client = _FakeConnectClient({"height": 100, "hex": header_hex})
|
||||
listener = ElectrumListener(lambda endpoint: client, session_factory, _ENDPOINTS)
|
||||
|
||||
subscribe_started = asyncio.Event()
|
||||
|
||||
async def blocked_subscribe_and_refresh(scripthash, user_id):
|
||||
subscribe_started.set()
|
||||
await asyncio.sleep(3600) # simulates a slow sweep; cancelled on cleanup
|
||||
|
||||
listener._subscribe_and_refresh = blocked_subscribe_and_refresh
|
||||
|
||||
run_once_task = asyncio.create_task(listener._run_once(_ENDPOINTS[0]))
|
||||
try:
|
||||
await asyncio.wait_for(subscribe_started.wait(), timeout=2)
|
||||
|
||||
# Resubscribing is still stuck mid-flight — but a new tip must still be
|
||||
# processed, proving the header consumer isn't blocked behind it.
|
||||
headers_queue = client.notifications("blockchain.headers.subscribe")
|
||||
next_header_hex = _mine_header(header_hex_to_block_hash(header_hex))
|
||||
await headers_queue.put([{"height": 101, "hex": next_header_hex}])
|
||||
await _wait_until(lambda: listener.tip_height == 101)
|
||||
|
||||
assert listener.tip_header_hex == next_header_hex
|
||||
finally:
|
||||
await client.close()
|
||||
await run_once_task
|
||||
|
||||
Reference in New Issue
Block a user