Add a periodic deposit reconciler, and stop losing subscribe tasks (B-30)

Deposits were credited exclusively by scripthash-change notifications,
with nothing re-verifying a user's balance against the chain if a
subscription was ever silently lost. address_for_new_user's subscribe
was fire-and-forget: the task wasn't retained, so it could be
garbage-collected mid-flight, and any failure (including self.client
turning None between the check and the task running) vanished into
asyncio's default unretrieved-exception handler instead of being
logged anywhere. On an otherwise healthy, long-lived connection there
may be no reconnect for days to re-subscribe everyone, so a user in
that state never saw their deposits.

address_for_new_user now retains the task and logs its exception if
it fails. New app/deposits/reconcile.py adds DepositReconciler, a
periodic sweep (every 5 minutes, gated on the Electrum client being
connected, same shape as tx/reconcile.py) that round-robins over every
user and calls the listener's own refresh_user (renamed from
_refresh_user since it's now called from outside the class) - so the
notification-driven and periodic paths can never behave differently.
Deliberately sweeps every user rather than only ones missing from the
in-memory scripthash map, since that map can't tell "never subscribed"
apart from "subscribed, but the server stopped delivering
notifications for it". Wired into app/main.py's lifespan alongside the
other three background reconcilers.

Suite grows from 176 to 182 tests. BUGS.md moves B-30 to Previously
fixed.
This commit is contained in:
davide committed 2026-07-27 10:35:23 +02:00
1 parent e8fdea0389
commit 63df38d30b
7 files changed
+255 -40

No files matched your search

+98
View File
@@ -0,0 +1,98 @@
"""Regression tests for B-30: a periodic sweep must catch a deposit whose
scripthash notification was silently lost, independent of whatever the
notification-driven path (electrum/listener.py:refresh_user) is doing."""
import pytest
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.db.base import Base
from app.db.models import User
from app.deposits.reconcile import DepositReconciler
@pytest.fixture
async def session_factory():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
yield async_sessionmaker(engine, expire_on_commit=False)
await engine.dispose()
async def _seed_users(session_factory, addresses: list[str]) -> list[int]:
async with session_factory() as session:
ids = []
for i, address in enumerate(addresses):
user = User(username=f"user{i}", password_hash="x", derivation_index=i, address=address)
session.add(user)
await session.flush()
ids.append(user.id)
await session.commit()
return ids
# Real, decodable PLM bech32 addresses (address_to_scripthash actually parses
# them) — arbitrary otherwise.
_ADDRESSES = [
"plm1q0xcqpzrky6eff2g52qdye53xkk9jxkvraxkkwd",
"plm1q5x25wd6q463mfhckjraaedgjg0lyu73qfcj43n",
"plm1qqph9qup2mp7w7g5nlsdhdc9m2pp44ampzw0ctx",
]
class FakeListener:
def __init__(self, *, fail_for: set[int] | None = None, disconnect_after: int | None = None):
self.client = object() # truthy: "connected"
self.refreshed: list[int] = []
self._fail_for = fail_for or set()
self._disconnect_after = disconnect_after
async def refresh_user(self, user_id: int, scripthash: str) -> None:
self.refreshed.append(user_id)
if self._disconnect_after is not None and len(self.refreshed) >= self._disconnect_after:
self.client = None
if user_id in self._fail_for:
raise RuntimeError(f"listunspent failed for user {user_id}")
async def test_sweep_once_refreshes_every_user(session_factory):
user_ids = await _seed_users(session_factory, _ADDRESSES)
listener = FakeListener()
reconciler = DepositReconciler(session_factory, listener)
await reconciler._sweep_once()
assert listener.refreshed == user_ids
async def test_sweep_once_continues_past_a_failing_user(session_factory):
"""One user's refresh failing (a transient network hiccup) must not stop the
sweep from reaching the rest — mirrors poll_once's per-item isolation."""
user_ids = await _seed_users(session_factory, _ADDRESSES)
listener = FakeListener(fail_for={user_ids[1]})
reconciler = DepositReconciler(session_factory, listener)
await reconciler._sweep_once()
assert listener.refreshed == user_ids
async def test_sweep_once_stops_when_the_connection_drops_mid_sweep(session_factory):
"""No point continuing once the connection is gone — the next reconnect's own
_subscribe_all_users will cover everyone anyway."""
user_ids = await _seed_users(session_factory, _ADDRESSES)
listener = FakeListener(disconnect_after=1)
reconciler = DepositReconciler(session_factory, listener)
await reconciler._sweep_once()
assert listener.refreshed == user_ids[:1]
async def test_sweep_once_does_nothing_with_no_users(session_factory):
listener = FakeListener()
reconciler = DepositReconciler(session_factory, listener)
await reconciler._sweep_once() # must not raise
assert listener.refreshed == []
+43 -7
View File
@@ -1,6 +1,7 @@
"""Listener-level behaviour: server rotation on failure (the fallback-servers
feature), the chain-tip monotonicity guard (B-19), header validation and
multi-server corroboration (B-28).
multi-server corroboration (B-28), and the new-user subscribe task's retention
and error logging (B-30).
The reconnect loop itself (B-01) is covered from the client side in
test_electrum_client.py — what's asserted here is that the listener *acts* on a
@@ -8,6 +9,7 @@ dead connection by moving to the next server instead of retrying the same one.
"""
import asyncio
import logging
import struct
import pytest
@@ -157,6 +159,40 @@ async def test_listener_with_no_endpoints_gives_up_loudly(session_factory):
assert listener.current_endpoint is None
# --- B-30: address_for_new_user's subscribe task must be retained (not fire-and-
# forget) and its failure must be observable, not silently swallowed. -------------
async def test_address_for_new_user_does_nothing_without_a_connection(session_factory):
listener = ElectrumListener(lambda endpoint: None, session_factory, _ENDPOINTS)
listener.address_for_new_user(1, "plm1q0xcqpzrky6eff2g52qdye53xkk9jxkvraxkkwd") # listener.client is None
assert listener._background_tasks == set()
async def test_address_for_new_user_retains_and_logs_a_failed_subscribe_task(session_factory, caplog):
"""Before B-30, this task was fire-and-forget: an AssertionError (self.client
turning None mid-flight) or any other failure vanished into asyncio's default
unretrieved-exception handler instead of being logged anywhere the operator
could see, and nothing kept the task alive in the meantime."""
class FailingClient:
async def subscribe_scripthash(self, scripthash):
raise ConnectionResetError("dropped mid-subscribe")
listener = ElectrumListener(lambda endpoint: None, session_factory, _ENDPOINTS)
listener.client = FailingClient()
listener.address_for_new_user(1, "plm1q0xcqpzrky6eff2g52qdye53xkk9jxkvraxkkwd")
assert len(listener._background_tasks) == 1 # retained while in flight
with caplog.at_level(logging.WARNING):
await asyncio.gather(*list(listener._background_tasks), return_exceptions=True)
await asyncio.sleep(0) # let the done_callbacks (scheduled via call_soon) run
assert listener._background_tasks == set() # discarded once done
assert "could not subscribe" in caplog.text
def test_tip_never_moves_backwards(session_factory):
"""B-19: `self.tip_height = header["height"]` accepted a lower height, and
_wait_for_next_block waits for tip_height > tip_at_close — so a regression
@@ -375,7 +411,7 @@ async def test_corroborate_utxo_spent_false_when_nobody_responds(session_factory
class _ActiveClient:
"""Stands in for `self.client`, the listener's one active connection —
_refresh_user only ever calls listunspent on it."""
refresh_user only ever calls listunspent on it."""
def __init__(self, entries: list[dict]):
self._entries = entries
@@ -406,8 +442,8 @@ async def _seed_funded_user(session_factory, *, username: str, address: str) ->
_UNRELATED_ENTRY = [{"tx_hash": "99" * 32, "tx_pos": 0, "height": 100, "value": 1_000_000}]
async def test_refresh_user_marks_a_utxo_spent_once_others_corroborate_it(session_factory):
user_id = await _seed_funded_user(session_factory, username="bob", address="plm1qtest")
async def testrefresh_user_marks_a_utxo_spent_once_others_corroborate_it(session_factory):
user_id = await _seed_funded_user(session_factory, username="bob", address="plm1q0xcqpzrky6eff2g52qdye53xkk9jxkvraxkkwd")
others_factory = await _listunspent_client_factory(
{"first.example": [], "second.example": [], "third.example": []}
@@ -415,7 +451,7 @@ async def test_refresh_user_marks_a_utxo_spent_once_others_corroborate_it(sessio
listener = ElectrumListener(others_factory, session_factory, _ENDPOINTS)
listener.client = _ActiveClient(_UNRELATED_ENTRY) # our own connection no longer sees the UTXO either
await listener._refresh_user(user_id, "scripthash")
await listener.refresh_user(user_id, "scripthash")
async with session_factory() as session:
utxo = (
@@ -428,7 +464,7 @@ async def test_refresh_user_marks_a_utxo_spent_once_others_corroborate_it(sessio
assert user.cached_balance_sats == 1_000_000
async def test_refresh_user_does_not_mark_when_corroboration_fails(session_factory):
async def testrefresh_user_does_not_mark_when_corroboration_fails(session_factory):
"""The single most important case: our own connection alone reporting the
UTXO missing must not be enough — before B-29 this zeroed the balance on one
bad reply."""
@@ -441,7 +477,7 @@ async def test_refresh_user_does_not_mark_when_corroboration_fails(session_facto
listener = ElectrumListener(others_factory, session_factory, _ENDPOINTS)
listener.client = _ActiveClient(_UNRELATED_ENTRY)
await listener._refresh_user(user_id, "scripthash")
await listener.refresh_user(user_id, "scripthash")
async with session_factory() as session:
utxo = (