Retry a stuck payout automatically, and log every failure (B-26)
_trigger_payout used to run exactly once, from _close_and_draw. Any failure after that point — no Electrum client, insufficient pool UTXOs, a missing fee_address, a rejected broadcast — wedged the round in paying_out forever, and every one of those early returns except the generic exception handler logged nothing at all: /admin showed a stalled round with no explanation. A process restart while paying_out hit the same dead end. _tick now handles status == "paying_out": it calls the new _retry_payout_if_due, which re-invokes _trigger_payout unless the most recent payout_failed audit entry for the round is younger than _PAYOUT_RETRY_INTERVAL_SECONDS (60s) — throttled so a persistently broken payout (e.g. no fee_address set yet) doesn't retry, and re-log a failure, on every 5-second tick. Every early return in _trigger_payout now calls _log_payout_failure with a reason string, so that throttle always has something to check against and /admin always shows why a round is stuck. This is safe to fire on a restart too, because B-25 already made _trigger_payout idempotent (it no-ops if a non-terminal payout PendingTransaction already exists) and persists before broadcasting — so a round found paying_out at startup, whatever state its payout was actually in, gets retried the same way. That closes the paying_out half of the "scheduler doesn't resume mid-flight rounds after a restart" gap in CLAUDE.md; the drawing/block-wait half is untouched (see BUGS.md B-36). BUGS.md moves B-26 to "Previously fixed" with the fix description; the suite grows from 143 to 148 tests. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -247,3 +247,109 @@ async def test_reserved_payout_outpoints_excludes_utxos_claimed_by_a_stale_payou
|
||||
reserved = await _reserved_payout_outpoints(session)
|
||||
|
||||
assert reserved == {(reserved_txid, 2)}
|
||||
|
||||
|
||||
# --- B-26: a "paying_out" round must retry its payout automatically ---------------
|
||||
|
||||
|
||||
async def test_trigger_payout_logs_a_failure_when_not_connected(payout_session_factory):
|
||||
"""Before B-26, this early return logged nothing beyond a log line — invisible
|
||||
in /admin and unusable as a signal for an automatic retry."""
|
||||
await _seed_paying_out_round(payout_session_factory)
|
||||
scheduler = RoundScheduler(payout_session_factory, FakePayoutListener(client=None))
|
||||
|
||||
await scheduler._trigger_payout(1)
|
||||
|
||||
async with payout_session_factory() as session:
|
||||
entries = (await session.scalars(select(AuditLog).where(AuditLog.event_type == "payout_failed"))).all()
|
||||
assert len(entries) == 1
|
||||
assert entries[0].payload_json.count("electrum client not connected") == 1
|
||||
|
||||
|
||||
async def test_trigger_payout_logs_a_failure_when_fee_address_missing(payout_session_factory):
|
||||
winner_id = await _seed_paying_out_round(payout_session_factory)
|
||||
async with payout_session_factory() as session:
|
||||
config = (await session.scalars(select(RoundConfig))).one()
|
||||
config.fee_address = ""
|
||||
await session.commit()
|
||||
|
||||
entries = [{"tx_hash": "77" * 32, "tx_pos": 0, "height": 10, "value": _POOL_AMOUNT_SATS + 100_000}]
|
||||
scheduler = RoundScheduler(payout_session_factory, FakePayoutListener(FakePayoutClient(entries)))
|
||||
|
||||
await scheduler._trigger_payout(1)
|
||||
|
||||
async with payout_session_factory() as session:
|
||||
entry = (await session.scalars(select(AuditLog).where(AuditLog.event_type == "payout_failed"))).one()
|
||||
assert "no fee_address configured" in entry.payload_json
|
||||
assert entry.user_id == winner_id
|
||||
|
||||
|
||||
async def test_tick_retries_a_stuck_paying_out_round_with_no_recent_failure(payout_session_factory):
|
||||
"""The scenario B-26 exists for: a round stuck in "paying_out" (a prior failure,
|
||||
or a process restart mid-payout) with no non-terminal PendingTransaction. A
|
||||
fresh tick must retry rather than leaving it wedged forever."""
|
||||
await _seed_paying_out_round(payout_session_factory)
|
||||
entries = [{"tx_hash": "88" * 32, "tx_pos": 0, "height": 10, "value": _POOL_AMOUNT_SATS + 100_000}]
|
||||
client = FakePayoutClient(entries)
|
||||
scheduler = RoundScheduler(payout_session_factory, FakePayoutListener(client))
|
||||
|
||||
await scheduler._tick()
|
||||
|
||||
assert client.broadcasted
|
||||
async with payout_session_factory() as session:
|
||||
round_ = await session.get(Round, 1)
|
||||
assert round_.payout_txid is not None
|
||||
pending = (await session.scalars(select(PendingTransaction))).one()
|
||||
assert pending.status == "pending"
|
||||
|
||||
|
||||
async def test_tick_throttles_retry_after_a_recent_payout_failure(payout_session_factory):
|
||||
"""A payout that just failed must not be retried on the very next tick, or a
|
||||
persistently-broken payout (e.g. no fee_address) would spam a retry — and a
|
||||
fresh payout_failed audit entry — every _TICK_INTERVAL_SECONDS."""
|
||||
await _seed_paying_out_round(payout_session_factory)
|
||||
async with payout_session_factory() as session:
|
||||
session.add(
|
||||
AuditLog(
|
||||
event_type="payout_failed",
|
||||
payload_json='{"round_id": 1, "reason": "insufficient pool UTXOs"}',
|
||||
round_id=1,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
entries = [{"tx_hash": "99" * 32, "tx_pos": 0, "height": 10, "value": _POOL_AMOUNT_SATS + 100_000}]
|
||||
client = FakePayoutClient(entries)
|
||||
scheduler = RoundScheduler(payout_session_factory, FakePayoutListener(client))
|
||||
|
||||
await scheduler._tick()
|
||||
|
||||
assert not client.broadcasted
|
||||
async with payout_session_factory() as session:
|
||||
assert (await session.scalars(select(PendingTransaction))).all() == []
|
||||
|
||||
|
||||
async def test_tick_retries_once_the_throttle_window_has_elapsed(payout_session_factory):
|
||||
await _seed_paying_out_round(payout_session_factory)
|
||||
async with payout_session_factory() as session:
|
||||
session.add(
|
||||
AuditLog(
|
||||
event_type="payout_failed",
|
||||
payload_json='{"round_id": 1, "reason": "insufficient pool UTXOs"}',
|
||||
round_id=1,
|
||||
created_at=datetime.now(timezone.utc) - timedelta(seconds=120),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
entries = [{"tx_hash": "aa" * 32, "tx_pos": 0, "height": 10, "value": _POOL_AMOUNT_SATS + 100_000}]
|
||||
client = FakePayoutClient(entries)
|
||||
scheduler = RoundScheduler(payout_session_factory, FakePayoutListener(client))
|
||||
|
||||
await scheduler._tick()
|
||||
|
||||
assert client.broadcasted
|
||||
async with payout_session_factory() as session:
|
||||
pending = (await session.scalars(select(PendingTransaction))).one()
|
||||
assert pending.status == "pending"
|
||||
|
||||
Reference in New Issue
Block a user