Make a stalled draw wait observable (B-36)
_wait_for_next_block had no timeout, no log, and no audit entry: a connection that stopped advancing the tip left a round silently frozen in "drawing" with nothing in /admin to explain why. Log progress periodically, write a draw_stalled audit entry past a threshold (a few block-time multiples), and surface the wait via a new Round.drawing_started_at column, exposed as draw_waiting_since in GET /rounds/current. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.timeutil import isoformat_utc
|
||||
from app.auth.dependencies import get_optional_user
|
||||
from app.db.models import RoundParticipant, User
|
||||
from app.db.session import get_session
|
||||
@@ -90,6 +91,10 @@ class CurrentRoundResponse(BaseModel):
|
||||
winner_amount_sats: int | None = None
|
||||
draw_block_height: int | None = None
|
||||
draw_block_hash: str | None = None
|
||||
# B-36: set only while status == "drawing", so the frontend can show "still
|
||||
# waiting for a block" rather than a countdown implying a bounded wait — this
|
||||
# phase has no timeout, only draw_animation_seconds' cosmetic minimum.
|
||||
draw_waiting_since: str | None = None
|
||||
chain_tip_height: int | None = None
|
||||
lottery_paused: bool = False
|
||||
user_played: bool = False
|
||||
@@ -168,6 +173,7 @@ async def current_round(
|
||||
winner_amount_sats=round_.winner_amount_sats,
|
||||
draw_block_height=round_.draw_block_height,
|
||||
draw_block_hash=round_.draw_block_hash,
|
||||
draw_waiting_since=isoformat_utc(round_.drawing_started_at) if round_.status == "drawing" else None,
|
||||
chain_tip_height=chain_tip_height,
|
||||
lottery_paused=config.paused,
|
||||
user_played=user_played,
|
||||
|
||||
@@ -72,6 +72,11 @@ class Round(Base):
|
||||
status: Mapped[str] = mapped_column(String(16), default="open")
|
||||
opened_at: Mapped[datetime] = mapped_column(default=utcnow)
|
||||
closed_at: Mapped[datetime | None] = mapped_column(default=None)
|
||||
# Set once, when status flips to "drawing" (rounds/scheduler.py:_close_and_draw).
|
||||
# Lets both the audit log (B-36's draw_stalled entries) and GET /rounds/current
|
||||
# (draw_waiting_since) measure how long a round has been waiting on a block,
|
||||
# since that wait has no timeout of its own — see _wait_for_next_block.
|
||||
drawing_started_at: Mapped[datetime | None] = mapped_column(default=None)
|
||||
draw_block_height: Mapped[int | None] = mapped_column(default=None)
|
||||
draw_block_hash: Mapped[str | None] = mapped_column(String(64), default=None)
|
||||
seed_int: Mapped[str | None] = mapped_column(String(128), default=None)
|
||||
|
||||
+49
-3
@@ -30,6 +30,13 @@ _TICK_INTERVAL_SECONDS = 5
|
||||
# (a dropped Electrum connection, a momentarily-empty pool) self-heals quickly.
|
||||
_PAYOUT_RETRY_INTERVAL_SECONDS = 60
|
||||
|
||||
# B-36: _wait_for_next_block has no timeout of its own — a round can legitimately
|
||||
# wait several PLM blocks (120s each) for its draw entropy, and re-waits on a
|
||||
# corroboration failure. These only make an already-long wait *observable*, they
|
||||
# never cut it short.
|
||||
_DRAW_PROGRESS_LOG_INTERVAL_SECONDS = 60
|
||||
_DRAW_STALL_THRESHOLD_SECONDS = 360 # a few multiples of PLM's 120s block time
|
||||
|
||||
|
||||
class RoundScheduler:
|
||||
"""Background task implementing flowchart.mmd's DRAW subgraph: closes the
|
||||
@@ -138,11 +145,13 @@ class RoundScheduler:
|
||||
user_by_address[user.address] = user.id
|
||||
|
||||
round_.status = "drawing"
|
||||
drawing_started_at = datetime.now(timezone.utc)
|
||||
round_.drawing_started_at = drawing_started_at
|
||||
await session.commit()
|
||||
broadcaster.publish()
|
||||
|
||||
tip_at_close = self._listener.tip_height
|
||||
block_height, block_hash = await self._wait_for_next_block(round_id, tip_at_close)
|
||||
block_height, block_hash = await self._wait_for_next_block(round_id, tip_at_close, drawing_started_at)
|
||||
winner_address = draw_winner(addresses, block_hash)
|
||||
|
||||
async with self._session_factory() as session:
|
||||
@@ -172,7 +181,9 @@ class RoundScheduler:
|
||||
logger.info("round %s: winner=%s pool=%s", round_id, winner_address, pool_amount)
|
||||
await self._trigger_payout(round_id)
|
||||
|
||||
async def _wait_for_next_block(self, round_id: int, tip_at_close: int) -> tuple[int, str]:
|
||||
async def _wait_for_next_block(
|
||||
self, round_id: int, tip_at_close: int, waiting_since: datetime
|
||||
) -> tuple[int, str]:
|
||||
"""Waits for a block after tip_at_close and, before handing it back as the
|
||||
draw's entropy source, requires it to be corroborated by the other
|
||||
configured Electrum servers (B-28) — our own active connection is exactly
|
||||
@@ -180,9 +191,44 @@ class RoundScheduler:
|
||||
not enough to seed a payout. A candidate that fails corroboration is never
|
||||
used: this keeps waiting for a further block and tries corroborating that
|
||||
one instead, logging why every time so a stuck draw is visible in
|
||||
/admin's audit log rather than a silent, unexplained wait."""
|
||||
/admin's audit log rather than a silent, unexplained wait.
|
||||
|
||||
This wait has no timeout — it can't, since the draw's entropy genuinely
|
||||
depends on a future block. B-36: what it lacked was *visibility*, so a
|
||||
connection that stopped advancing the tip left the round silently frozen
|
||||
in "drawing" with nothing in the logs or /admin to explain why. Progress
|
||||
is now logged periodically, and past _DRAW_STALL_THRESHOLD_SECONDS a
|
||||
draw_stalled audit entry is written (and re-written every threshold
|
||||
interval for as long as the stall continues) so the wait shows up next
|
||||
to the draw_header_corroboration_failed entries above.
|
||||
"""
|
||||
next_progress_log_at = waiting_since + timedelta(seconds=_DRAW_PROGRESS_LOG_INTERVAL_SECONDS)
|
||||
next_stall_audit_at = waiting_since + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
||||
while True:
|
||||
while self._listener.tip_height <= tip_at_close or not self._listener.tip_header_hex:
|
||||
now = datetime.now(timezone.utc)
|
||||
if now >= next_progress_log_at:
|
||||
logger.info(
|
||||
"round %s: still waiting for a block past height %s (%.0fs since drawing started)",
|
||||
round_id,
|
||||
tip_at_close,
|
||||
(now - waiting_since).total_seconds(),
|
||||
)
|
||||
next_progress_log_at = now + timedelta(seconds=_DRAW_PROGRESS_LOG_INTERVAL_SECONDS)
|
||||
if now >= next_stall_audit_at:
|
||||
async with self._session_factory() as session:
|
||||
await write_audit_log(
|
||||
session,
|
||||
"draw_stalled",
|
||||
{
|
||||
"tip_at_close": tip_at_close,
|
||||
"current_tip_height": self._listener.tip_height,
|
||||
"elapsed_seconds": int((now - waiting_since).total_seconds()),
|
||||
},
|
||||
round_id=round_id,
|
||||
)
|
||||
await session.commit()
|
||||
next_stall_audit_at = now + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
||||
await asyncio.sleep(_TICK_INTERVAL_SECONDS)
|
||||
height = self._listener.tip_height
|
||||
block_hash = header_hex_to_block_hash(self._listener.tip_header_hex)
|
||||
|
||||
Reference in New Issue
Block a user