Make a stalled draw wait observable (B-36)
_wait_for_next_block had no timeout, no log, and no audit entry: a connection that stopped advancing the tip left a round silently frozen in "drawing" with nothing in /admin to explain why. Log progress periodically, write a draw_stalled audit entry past a threshold (a few block-time multiples), and surface the wait via a new Round.drawing_started_at column, exposed as draw_waiting_since in GET /rounds/current. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+49
-3
@@ -30,6 +30,13 @@ _TICK_INTERVAL_SECONDS = 5
|
||||
# (a dropped Electrum connection, a momentarily-empty pool) self-heals quickly.
|
||||
_PAYOUT_RETRY_INTERVAL_SECONDS = 60
|
||||
|
||||
# B-36: _wait_for_next_block has no timeout of its own — a round can legitimately
|
||||
# wait several PLM blocks (120s each) for its draw entropy, and re-waits on a
|
||||
# corroboration failure. These only make an already-long wait *observable*, they
|
||||
# never cut it short.
|
||||
_DRAW_PROGRESS_LOG_INTERVAL_SECONDS = 60
|
||||
_DRAW_STALL_THRESHOLD_SECONDS = 360 # a few multiples of PLM's 120s block time
|
||||
|
||||
|
||||
class RoundScheduler:
|
||||
"""Background task implementing flowchart.mmd's DRAW subgraph: closes the
|
||||
@@ -138,11 +145,13 @@ class RoundScheduler:
|
||||
user_by_address[user.address] = user.id
|
||||
|
||||
round_.status = "drawing"
|
||||
drawing_started_at = datetime.now(timezone.utc)
|
||||
round_.drawing_started_at = drawing_started_at
|
||||
await session.commit()
|
||||
broadcaster.publish()
|
||||
|
||||
tip_at_close = self._listener.tip_height
|
||||
block_height, block_hash = await self._wait_for_next_block(round_id, tip_at_close)
|
||||
block_height, block_hash = await self._wait_for_next_block(round_id, tip_at_close, drawing_started_at)
|
||||
winner_address = draw_winner(addresses, block_hash)
|
||||
|
||||
async with self._session_factory() as session:
|
||||
@@ -172,7 +181,9 @@ class RoundScheduler:
|
||||
logger.info("round %s: winner=%s pool=%s", round_id, winner_address, pool_amount)
|
||||
await self._trigger_payout(round_id)
|
||||
|
||||
async def _wait_for_next_block(self, round_id: int, tip_at_close: int) -> tuple[int, str]:
|
||||
async def _wait_for_next_block(
|
||||
self, round_id: int, tip_at_close: int, waiting_since: datetime
|
||||
) -> tuple[int, str]:
|
||||
"""Waits for a block after tip_at_close and, before handing it back as the
|
||||
draw's entropy source, requires it to be corroborated by the other
|
||||
configured Electrum servers (B-28) — our own active connection is exactly
|
||||
@@ -180,9 +191,44 @@ class RoundScheduler:
|
||||
not enough to seed a payout. A candidate that fails corroboration is never
|
||||
used: this keeps waiting for a further block and tries corroborating that
|
||||
one instead, logging why every time so a stuck draw is visible in
|
||||
/admin's audit log rather than a silent, unexplained wait."""
|
||||
/admin's audit log rather than a silent, unexplained wait.
|
||||
|
||||
This wait has no timeout — it can't, since the draw's entropy genuinely
|
||||
depends on a future block. B-36: what it lacked was *visibility*, so a
|
||||
connection that stopped advancing the tip left the round silently frozen
|
||||
in "drawing" with nothing in the logs or /admin to explain why. Progress
|
||||
is now logged periodically, and past _DRAW_STALL_THRESHOLD_SECONDS a
|
||||
draw_stalled audit entry is written (and re-written every threshold
|
||||
interval for as long as the stall continues) so the wait shows up next
|
||||
to the draw_header_corroboration_failed entries above.
|
||||
"""
|
||||
next_progress_log_at = waiting_since + timedelta(seconds=_DRAW_PROGRESS_LOG_INTERVAL_SECONDS)
|
||||
next_stall_audit_at = waiting_since + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
||||
while True:
|
||||
while self._listener.tip_height <= tip_at_close or not self._listener.tip_header_hex:
|
||||
now = datetime.now(timezone.utc)
|
||||
if now >= next_progress_log_at:
|
||||
logger.info(
|
||||
"round %s: still waiting for a block past height %s (%.0fs since drawing started)",
|
||||
round_id,
|
||||
tip_at_close,
|
||||
(now - waiting_since).total_seconds(),
|
||||
)
|
||||
next_progress_log_at = now + timedelta(seconds=_DRAW_PROGRESS_LOG_INTERVAL_SECONDS)
|
||||
if now >= next_stall_audit_at:
|
||||
async with self._session_factory() as session:
|
||||
await write_audit_log(
|
||||
session,
|
||||
"draw_stalled",
|
||||
{
|
||||
"tip_at_close": tip_at_close,
|
||||
"current_tip_height": self._listener.tip_height,
|
||||
"elapsed_seconds": int((now - waiting_since).total_seconds()),
|
||||
},
|
||||
round_id=round_id,
|
||||
)
|
||||
await session.commit()
|
||||
next_stall_audit_at = now + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
||||
await asyncio.sleep(_TICK_INTERVAL_SECONDS)
|
||||
height = self._listener.tip_height
|
||||
block_hash = header_hex_to_block_hash(self._listener.tip_header_hex)
|
||||
|
||||
Reference in New Issue
Block a user