Add a periodic deposit reconciler, and stop losing subscribe tasks (B-30)
Deposits were credited exclusively by scripthash-change notifications, with nothing re-verifying a user's balance against the chain if a subscription was ever silently lost. address_for_new_user's subscribe was fire-and-forget: the task wasn't retained, so it could be garbage-collected mid-flight, and any failure (including self.client turning None between the check and the task running) vanished into asyncio's default unretrieved-exception handler instead of being logged anywhere. On an otherwise healthy, long-lived connection there may be no reconnect for days to re-subscribe everyone, so a user in that state never saw their deposits. address_for_new_user now retains the task and logs its exception if it fails. New app/deposits/reconcile.py adds DepositReconciler, a periodic sweep (every 5 minutes, gated on the Electrum client being connected, same shape as tx/reconcile.py) that round-robins over every user and calls the listener's own refresh_user (renamed from _refresh_user since it's now called from outside the class) - so the notification-driven and periodic paths can never behave differently. Deliberately sweeps every user rather than only ones missing from the in-memory scripthash map, since that map can't tell "never subscribed" apart from "subscribed, but the server stopped delivering notifications for it". Wired into app/main.py's lifespan alongside the other three background reconcilers. Suite grows from 176 to 182 tests. BUGS.md moves B-30 to Previously fixed.
This commit is contained in:
1 parent
e8fdea0389
commit
63df38d30b
7 files changed
+255
-40
No files matched your search
@@ -0,0 +1,69 @@
|
||||
"""Periodic safety net for deposit crediting and external-spend detection (B-30),
|
||||
independent of scripthash-change notifications.
|
||||
|
||||
Those notifications are the fast path, but nothing else re-verifies a user's
|
||||
balance against the chain if one is ever silently lost: `address_for_new_user`'s
|
||||
subscribe is best-effort (its own failure just logs, see electrum/listener.py),
|
||||
and on an otherwise healthy, long-lived connection there may be no reconnect for
|
||||
days — the only other event that re-subscribes everyone from scratch. Without
|
||||
this, a single lost subscription meant that user's deposits were never credited,
|
||||
indefinitely.
|
||||
|
||||
This mirrors app/tx/reconcile.py's shape (a periodic sweep gated on the Electrum
|
||||
client being connected) but reuses ElectrumListener.refresh_user directly rather
|
||||
than re-implementing crediting/spend-detection, so the notification-driven and
|
||||
periodic paths can never behave differently from each other.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from app.db.models import User
|
||||
from app.electrum.listener import ElectrumListener
|
||||
from app.electrum.scripthash import address_to_scripthash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SWEEP_INTERVAL_SECONDS = 300
|
||||
|
||||
|
||||
class DepositReconciler:
|
||||
def __init__(self, session_factory: async_sessionmaker, listener: ElectrumListener):
|
||||
self._session_factory = session_factory
|
||||
self._listener = listener
|
||||
|
||||
async def run(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(_SWEEP_INTERVAL_SECONDS)
|
||||
if self._listener.client is None:
|
||||
continue
|
||||
try:
|
||||
await self._sweep_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("deposit reconciliation sweep failed")
|
||||
|
||||
async def _sweep_once(self) -> None:
|
||||
"""Round-robins over every user's address rather than only ones missing
|
||||
from the listener's in-memory `_scripthash_to_user` map: that map can't
|
||||
tell "never subscribed" apart from "subscribed, but this server silently
|
||||
stopped delivering notifications for it" — exactly the failure mode this
|
||||
exists to catch. One user failing (a transient network hiccup) must not
|
||||
stop the sweep from reaching the rest, mirroring poll_once's per-item
|
||||
isolation in tx/confirmation.py.
|
||||
"""
|
||||
async with self._session_factory() as session:
|
||||
users = (await session.scalars(select(User))).all()
|
||||
|
||||
for user in users:
|
||||
if self._listener.client is None:
|
||||
return # connection dropped mid-sweep; the next reconnect's own _subscribe_all_users covers everyone
|
||||
scripthash = address_to_scripthash(user.address)
|
||||
try:
|
||||
await self._listener.refresh_user(user.id, scripthash)
|
||||
except Exception:
|
||||
logger.exception("deposit reconciliation failed for user_id=%s", user.id)
|
||||
@@ -113,7 +113,7 @@ async def find_utxos_missing_from(session: AsyncSession, user_id: int, entries:
|
||||
Returning a row here is *not* proof it was actually spent — only that this one
|
||||
server's reply no longer lists it. A single broken, behind, or malicious
|
||||
server could otherwise zero a user's balance on one bad reply, which is why
|
||||
the caller (electrum/listener.py:_refresh_user) must independently
|
||||
the caller (electrum/listener.py:refresh_user) must independently
|
||||
corroborate each candidate against other configured servers before treating
|
||||
it as genuine, rather than this function marking anything itself.
|
||||
|
||||
|
||||
@@ -63,6 +63,11 @@ class ElectrumListener:
|
||||
self._endpoints = list(endpoints or [])
|
||||
self._endpoint_index = 0
|
||||
self._scripthash_to_user: dict[str, int] = {}
|
||||
# Retains address_for_new_user's fire-and-forget subscribe task so it
|
||||
# can't be garbage-collected mid-flight, and so its exception (if any) is
|
||||
# actually observed instead of only reaching asyncio's default "Task
|
||||
# exception was never retrieved" handler (B-30).
|
||||
self._background_tasks: set[asyncio.Task] = set()
|
||||
self.tip_height: int = 0
|
||||
self.tip_header_hex: str | None = None
|
||||
self.client: ElectrumClient | None = None
|
||||
@@ -77,11 +82,32 @@ class ElectrumListener:
|
||||
|
||||
def address_for_new_user(self, user_id: int, address: str) -> None:
|
||||
"""Called right after a user registers so their deposit address starts
|
||||
being watched immediately, without waiting for the next reconnect cycle."""
|
||||
being watched immediately, without waiting for the next reconnect cycle.
|
||||
|
||||
Best-effort, not retried on its own: `self.client` can still become None
|
||||
between the check below and the task actually running (the connection
|
||||
drops in between), which used to raise an AssertionError inside an
|
||||
untracked task and vanish silently (B-30). The exception is now logged
|
||||
instead, and — since a failure here just means this one address stays
|
||||
unsubscribed until the next reconnect's `_subscribe_all_users` or the
|
||||
periodic `DepositReconciler` sweep (also B-30) catches it — that's an
|
||||
acceptable, self-healing outcome rather than something worth its own
|
||||
retry/backoff loop.
|
||||
"""
|
||||
scripthash = address_to_scripthash(address)
|
||||
self._scripthash_to_user[scripthash] = user_id
|
||||
if self.client is not None:
|
||||
asyncio.create_task(self._subscribe_and_refresh(scripthash, user_id))
|
||||
task = asyncio.create_task(self._subscribe_and_refresh(scripthash, user_id))
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._background_tasks.discard)
|
||||
task.add_done_callback(self._log_subscribe_task_failure)
|
||||
|
||||
def _log_subscribe_task_failure(self, task: asyncio.Task) -> None:
|
||||
if task.cancelled():
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc is not None:
|
||||
logger.warning("could not subscribe a newly-registered user's address: %r", exc)
|
||||
|
||||
async def run(self) -> None:
|
||||
backoff = 1
|
||||
@@ -179,7 +205,7 @@ class ElectrumListener:
|
||||
async def _subscribe_and_refresh(self, scripthash: str, user_id: int) -> None:
|
||||
assert self.client is not None
|
||||
await self.client.subscribe_scripthash(scripthash)
|
||||
await self._refresh_user(user_id, scripthash)
|
||||
await self.refresh_user(user_id, scripthash)
|
||||
|
||||
def _apply_header(self, header: dict) -> None:
|
||||
"""Record a new chain tip, refusing to move backwards.
|
||||
@@ -329,9 +355,9 @@ class ElectrumListener:
|
||||
scripthash, _status = await queue.get()
|
||||
user_id = self._scripthash_to_user.get(scripthash)
|
||||
if user_id is not None:
|
||||
await self._refresh_user(user_id, scripthash)
|
||||
await self.refresh_user(user_id, scripthash)
|
||||
|
||||
async def _refresh_user(self, user_id: int, scripthash: str) -> None:
|
||||
async def refresh_user(self, user_id: int, scripthash: str) -> None:
|
||||
"""Three phases, so no DB session is held across a network call (B-18),
|
||||
same shape as _trigger_payout: read what's needed, corroborate any
|
||||
candidate external spends against other servers (B-29), then persist.
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.auth.routes import router as auth_router
|
||||
from app.api.errors import ApiError
|
||||
from app.config import settings, validate_runtime_secrets
|
||||
from app.db.base import AsyncSessionLocal
|
||||
from app.deposits.reconcile import DepositReconciler
|
||||
from app.electrum.client import ElectrumClient, ElectrumEndpoint, parse_endpoints
|
||||
from app.electrum.listener import ElectrumListener
|
||||
from app.rounds.scheduler import RoundScheduler
|
||||
@@ -61,6 +62,10 @@ async def lifespan(app: FastAPI):
|
||||
# Resolves in-flight transactions against the chain — the piece that lets the
|
||||
# system recover on its own from a broadcast that never confirmed (B-04/B-08).
|
||||
reconciler = PendingTransactionReconciler(AsyncSessionLocal, lambda: listener.client)
|
||||
# Periodic safety net for deposit crediting/external-spend detection,
|
||||
# independent of scripthash-change notifications — catches a subscription
|
||||
# silently lost on an otherwise healthy connection (B-30).
|
||||
deposit_reconciler = DepositReconciler(AsyncSessionLocal, listener)
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(listener.run()),
|
||||
@@ -68,6 +73,7 @@ async def lifespan(app: FastAPI):
|
||||
asyncio.create_task(poller.run()),
|
||||
asyncio.create_task(bumper.run()),
|
||||
asyncio.create_task(reconciler.run()),
|
||||
asyncio.create_task(deposit_reconciler.run()),
|
||||
]
|
||||
try:
|
||||
yield
|
||||
|
||||
Reference in new issue
Block a user