davideandClaude Sonnet 5.5
575f1a5354
- GET /health/ready (Electrum connected, header < 1800s, scheduler heartbeat < 30s) is the container healthcheck; the scheduler beats on every tick and draw-wait iteration - GET /admin/electrum and an Electrum indicator in /admin (server, header age, last rotation) - GET /admin/audit-log: before_id, event_type, round_id, user_id, from, to; the panel gets filters - registration writes user_registered in its transaction - the app's own logs also go to stdout; uvicorn trusts X-Forwarded-For from Caddy's fixed address only, so the access log shows the client's IP Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
863 lines
45 KiB
Python
863 lines
45 KiB
Python
import asyncio
|
|
import time
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from embit import script
|
|
from embit.transaction import Transaction
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.exc import OperationalError
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
|
|
from app.audit.log import write_audit_log
|
|
from app.db.base import begin_immediate
|
|
from app.db.models import AuditLog, PendingTransaction, Round, RoundParticipant, User
|
|
from app.electrum.listener import ElectrumListener
|
|
from app.electrum.scripthash import address_to_scripthash
|
|
from app.rounds.config import get_round_config
|
|
from app.rounds.draw import (
|
|
draw_index,
|
|
draw_seed,
|
|
new_server_seed,
|
|
participants_hash,
|
|
server_seed_commitment,
|
|
)
|
|
from app.rounds.events import broadcaster
|
|
from app.rounds.pause import pause_automatically
|
|
from app.rounds.service import open_new_round_if_needed, round_deadline, winner_share
|
|
from app.tx.outcome import BroadcastOutcome, broadcast_with_verdict
|
|
from app.tx.versions import record_version
|
|
from app.wallet.hd import derive_pool_address, derive_round_pool_key
|
|
from app.wallet.keystore import decrypt_round_seed, encrypt_round_seed
|
|
from app.wallet.plm_network import PLM_MAINNET
|
|
from app.wallet.psbt_builder import InsufficientFundsError, Utxo, build_payout_transaction
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_TICK_INTERVAL_SECONDS = 5
|
|
|
|
# B-26: how long to wait after a payout failure before automatically retrying it.
|
|
# Long enough that a persistently-broken payout (misconfigured fee_address,
|
|
# insufficient pool UTXOs) doesn't re-attempt — and re-write a payout_failed audit
|
|
# entry — every _TICK_INTERVAL_SECONDS; short enough that a transient failure
|
|
# (a dropped Electrum connection, a momentarily-empty pool) self-heals quickly.
|
|
_PAYOUT_RETRY_INTERVAL_SECONDS = 60
|
|
|
|
# B-36: _wait_for_next_block has no timeout of its own — a round can legitimately
|
|
# wait several PLM blocks (120s each) for its draw entropy, and re-waits on a
|
|
# corroboration failure. These only make an already-long wait *observable*, they
|
|
# never cut it short.
|
|
_DRAW_PROGRESS_LOG_INTERVAL_SECONDS = 60
|
|
# B-96 (TX-29, AUD-46): how long past its deadline a round may sit in `closing` on an in-flight bet before
|
|
# `closing_stalled` is written, and how often it repeats. The wait itself has no limit (ADR-51).
|
|
CLOSING_STALL_THRESHOLD_SECONDS = 1800
|
|
_DRAW_STALL_THRESHOLD_SECONDS = 360 # a few multiples of PLM's 120s block time
|
|
|
|
|
|
class RoundScheduler:
|
|
"""Background task implementing the round lifecycle (docs/ciclo-di-vita-del-round.md):
|
|
closes the round on its timer (once any in-flight bets have confirmed), draws a
|
|
winner from the next confirmed block, and broadcasts the payout. The next round
|
|
only opens once this one is fully closed (rounds/service.get_active_round)."""
|
|
|
|
def __init__(self, session_factory: async_sessionmaker, listener: ElectrumListener):
|
|
self._session_factory = session_factory
|
|
self._listener = listener
|
|
# B-92 (ARC-61): monotonic time of the last tick or draw-wait iteration, read by
|
|
# /health/ready. The draw wait runs inside a tick and lasts several blocks, so it beats too.
|
|
self.last_heartbeat: float | None = None
|
|
|
|
def heartbeat_age_seconds(self) -> int | None:
|
|
return None if self.last_heartbeat is None else int(time.monotonic() - self.last_heartbeat)
|
|
|
|
async def run(self) -> None:
|
|
while True:
|
|
self.last_heartbeat = time.monotonic()
|
|
try:
|
|
await self._tick()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.exception("round scheduler tick failed")
|
|
await asyncio.sleep(_TICK_INTERVAL_SECONDS)
|
|
|
|
async def _tick(self) -> None:
|
|
if self._listener.client is None:
|
|
return
|
|
|
|
async with self._session_factory() as session:
|
|
# B-88: every state transition below starts with BEGIN IMMEDIATE (DAT-74).
|
|
# A busy database raises out of the tick, which run() logs and retries at
|
|
# the next one — nothing has been written yet.
|
|
await begin_immediate(session)
|
|
round_ = await open_new_round_if_needed(session)
|
|
await session.commit()
|
|
if round_ is None:
|
|
return # still in the cooldown window after the last round closed
|
|
round_id, status = round_.id, round_.status
|
|
# B-61: this round's own snapshotted deadline, not one recomputed from
|
|
# whatever the config says now — an operator lowering the duration
|
|
# mid-round used to close the round on the spot.
|
|
deadline = round_deadline(round_)
|
|
|
|
if status == "paying_out":
|
|
# B-26: _trigger_payout used to run exactly once, from _close_and_draw —
|
|
# any failure after that (no Electrum client, insufficient pool UTXOs, a
|
|
# rejected broadcast) or a process restart while paying_out left the round
|
|
# wedged here forever. Every tick now re-checks and retries, throttled by
|
|
# _retry_payout_if_due so a persistent failure doesn't retry on every tick.
|
|
await self._retry_payout_if_due(round_id)
|
|
return
|
|
|
|
if status == "drawing":
|
|
# B-81 (RND-36, ARC-27): a round in `drawing` is picked up here at every step, not
|
|
# only from inside the _close_and_draw call that entered it. Everything the draw
|
|
# needs — secret, ordered list with positions, fingerprint, height, confirmations
|
|
# — is in the database since the transition (B-73), so an exception in an earlier
|
|
# step (a locked database while writing an audit row, a malformed header) or a
|
|
# restart leaves nothing behind that this could not rebuild, and the resumed draw
|
|
# gives the same winner. The wait stays inside this step: nothing else can run
|
|
# until the round is closed anyway, since no other round may open.
|
|
await self._complete_draw(round_id)
|
|
return
|
|
|
|
if status not in ("open", "closing"):
|
|
return
|
|
|
|
if status == "open":
|
|
if datetime.now(timezone.utc) < deadline:
|
|
return
|
|
|
|
async with self._session_factory() as session:
|
|
await begin_immediate(session) # B-88
|
|
round_ = await session.get(Round, round_id)
|
|
# No new bets from here on, regardless of how long the pending-bet
|
|
# wait below takes — flip to "closing" immediately so it's observable
|
|
# via /rounds/current (e.g. "round closed, waiting for jackpot
|
|
# confirmation") instead of silently staying "open" past the deadline.
|
|
round_.status = "closing"
|
|
round_.closed_at = datetime.now(timezone.utc)
|
|
await session.commit()
|
|
broadcaster.publish()
|
|
|
|
async with self._session_factory() as session:
|
|
# "building" counts as in-flight too: it's a bet mid-broadcast (see
|
|
# bets/service.py's two-phase write). A bet that never confirms is
|
|
# eventually removed by app/tx/reconcile.py, which is what stops this
|
|
# wait from being unbounded.
|
|
pending_count = await session.scalar(
|
|
select(func.count())
|
|
.select_from(RoundParticipant)
|
|
.where(
|
|
RoundParticipant.round_id == round_id,
|
|
RoundParticipant.status.in_(("building", "broadcast")),
|
|
)
|
|
)
|
|
if pending_count:
|
|
await self._note_closing_stall(round_id, deadline)
|
|
return # wait for in-flight bets to confirm before closing; stays "closing"
|
|
|
|
await self._close_and_draw(round_id)
|
|
|
|
async def _note_closing_stall(self, round_id: int, deadline: datetime) -> None:
|
|
"""B-96 (TX-29, AUD-46): the wait in `closing` is unbounded by design, so it is made
|
|
visible — one `closing_stalled` row per CLOSING_STALL_THRESHOLD_SECONDS once the round has
|
|
been past its deadline for that long. The latest row is the clock of the next one (read
|
|
from the log, not from memory, so a restart neither repeats nor anticipates it)."""
|
|
now = datetime.now(timezone.utc)
|
|
seconds_past = (now - deadline).total_seconds()
|
|
if seconds_past < CLOSING_STALL_THRESHOLD_SECONDS:
|
|
return
|
|
async with self._session_factory() as session:
|
|
last = await session.scalar(
|
|
select(AuditLog.created_at)
|
|
.where(AuditLog.event_type == "closing_stalled", AuditLog.round_id == round_id)
|
|
.order_by(AuditLog.id.desc())
|
|
.limit(1)
|
|
)
|
|
if last is not None and now < last.replace(tzinfo=timezone.utc) + timedelta(
|
|
seconds=CLOSING_STALL_THRESHOLD_SECONDS
|
|
):
|
|
return
|
|
txids = list(
|
|
await session.scalars(
|
|
select(RoundParticipant.bet_txid)
|
|
.where(
|
|
RoundParticipant.round_id == round_id,
|
|
RoundParticipant.status.in_(("building", "broadcast")),
|
|
)
|
|
.order_by(RoundParticipant.id)
|
|
)
|
|
)
|
|
await write_audit_log(
|
|
session,
|
|
"closing_stalled",
|
|
{"in_flight": len(txids), "txids": txids, "seconds_past_deadline": int(seconds_past)},
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
logger.warning("round %s has waited %ss past its deadline for %s in-flight bet(s)", round_id, int(seconds_past), len(txids))
|
|
|
|
async def _close_and_draw(self, round_id: int) -> None:
|
|
"""Close a `closing` round once nothing is in flight: either close it (no confirmed
|
|
participant) or commit to the draw — fix the participant list, its fingerprint and the
|
|
draw height, all in one transaction, and enter `drawing` (RND-13, RND-14, RND-33,
|
|
DAT-65) — then run the draw itself (`_complete_draw`)."""
|
|
async with self._session_factory() as session:
|
|
await begin_immediate(session) # B-88
|
|
round_ = await session.get(Round, round_id)
|
|
|
|
# B-53: re-check for in-flight bets in the *same* session that snapshots
|
|
# the participants. _tick's check ran in a session of its own, so a bet
|
|
# committing its "building" row in between was counted by neither: the
|
|
# round drew and paid out without it, while its sats still landed in the
|
|
# pool. place_bet's compare-and-set on the round row is what makes that
|
|
# window unreachable; this is the cheap second lock on the same door, and
|
|
# it fails safe — the round stays "closing" and the next tick retries.
|
|
pending_count = await session.scalar(
|
|
select(func.count())
|
|
.select_from(RoundParticipant)
|
|
.where(
|
|
RoundParticipant.round_id == round_id,
|
|
RoundParticipant.status.in_(("building", "broadcast")),
|
|
)
|
|
)
|
|
if pending_count:
|
|
logger.info(
|
|
"round %s: %s bet(s) still in flight at close time, waiting", round_id, pending_count
|
|
)
|
|
return
|
|
|
|
# B-73 (RND-14, DAT-66): the draw order is the bet txid, ascending — a value the
|
|
# public can read off the chain, with no ties (two bets never share a txid). The
|
|
# server-only `broadcast_at` it replaces could not be checked by anyone.
|
|
participants = (
|
|
await session.scalars(
|
|
select(RoundParticipant)
|
|
.where(RoundParticipant.round_id == round_id, RoundParticipant.status == "confirmed")
|
|
.order_by(RoundParticipant.bet_txid)
|
|
)
|
|
).all()
|
|
|
|
if not participants:
|
|
round_.status = "closed"
|
|
# DAT-64: nothing was drawn, so the secret has nothing left to protect.
|
|
if round_.server_seed_enc and not round_.server_seed:
|
|
round_.server_seed = decrypt_round_seed(round_.server_seed_enc)
|
|
await write_audit_log(session, "round_closed", {"participants": 0}, round_id=round_id)
|
|
await session.commit()
|
|
broadcaster.publish()
|
|
logger.info("round %s closed with no participants", round_id)
|
|
return
|
|
|
|
# B-73 (RND-19, RND-33): draw_height is "the height we knew at closing, plus one", so
|
|
# an unknown height (0) cannot be a base — the round stays "closing" and this is
|
|
# tried again at the next step, with one audit row per round.
|
|
tip_at_close = self._listener.tip_height
|
|
if tip_at_close <= 0:
|
|
already_written = await session.scalar(
|
|
select(func.count())
|
|
.select_from(AuditLog)
|
|
.where(AuditLog.event_type == "draw_baseline_tip_unknown", AuditLog.round_id == round_id)
|
|
)
|
|
if not already_written:
|
|
logger.warning("round %s: chain tip unknown at closing time, not entering drawing yet", round_id)
|
|
await write_audit_log(session, "draw_baseline_tip_unknown", {}, round_id=round_id)
|
|
await session.commit()
|
|
return
|
|
|
|
users = {
|
|
user.id: user
|
|
for user in (
|
|
await session.scalars(select(User).where(User.id.in_({p.user_id for p in participants})))
|
|
).all()
|
|
}
|
|
if not round_.server_seed_enc:
|
|
# A round opened before B-73 has no commitment. Give it a secret now: it still
|
|
# cannot be known to anyone who could steer the block, because the list and the
|
|
# height are fixed below before that block exists — only the early publication
|
|
# of the commitment is missing for this one round.
|
|
seed = new_server_seed()
|
|
round_.server_seed_hash = server_seed_commitment(seed)
|
|
round_.server_seed_enc = encrypt_round_seed(seed)
|
|
for position, participant in enumerate(participants):
|
|
participant.draw_position = position
|
|
round_.participants_hash = participants_hash(
|
|
[(p.bet_txid, p.bet_vout, users[p.user_id].address) for p in participants]
|
|
)
|
|
round_.draw_height = tip_at_close + 1
|
|
round_.pool_amount_sats = sum(p.bet_amount_sats for p in participants)
|
|
round_.status = "drawing"
|
|
round_.drawing_started_at = datetime.now(timezone.utc)
|
|
await write_audit_log(
|
|
session,
|
|
"draw_committed",
|
|
{
|
|
"participants_hash": round_.participants_hash,
|
|
"draw_height": round_.draw_height,
|
|
"participants": len(participants),
|
|
"server_seed_hash": round_.server_seed_hash,
|
|
},
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
broadcaster.publish()
|
|
|
|
await self._complete_draw(round_id)
|
|
|
|
async def _complete_draw(self, round_id: int) -> None:
|
|
"""The draw of a round in `drawing`, from nothing but what the database holds (RND-36,
|
|
DAT-65): wait for the block at `draw_height` to be confirmed and agreed on by the
|
|
quorum, compute the winner, store it and reveal the secret in one transaction, then
|
|
start the payout. Idempotent — a round no longer in `drawing`, or with a winner
|
|
already stored, is left alone."""
|
|
async with self._session_factory() as session:
|
|
round_ = await session.get(Round, round_id)
|
|
if round_ is None or round_.status != "drawing" or round_.winner_user_id is not None:
|
|
return
|
|
draw_height = round_.draw_height
|
|
draw_confirmations = round_.draw_confirmations
|
|
committed_hash = round_.participants_hash
|
|
server_seed_enc = round_.server_seed_enc
|
|
server_seed_hash = round_.server_seed_hash
|
|
waiting_since = round_.drawing_started_at.replace(tzinfo=timezone.utc)
|
|
|
|
block_hash = await self._wait_for_draw_block(round_id, draw_height, draw_confirmations, waiting_since)
|
|
|
|
server_seed = decrypt_round_seed(server_seed_enc)
|
|
if server_seed_commitment(server_seed) != server_seed_hash:
|
|
raise RuntimeError(f"round {round_id}: the stored secret does not match its published commitment")
|
|
|
|
async with self._session_factory() as session:
|
|
await begin_immediate(session) # B-88
|
|
round_ = await session.get(Round, round_id)
|
|
if round_.status != "drawing" or round_.winner_user_id is not None:
|
|
return # drawn meanwhile (a second pass over the same round)
|
|
participants = (
|
|
await session.scalars(
|
|
select(RoundParticipant)
|
|
.where(RoundParticipant.round_id == round_id, RoundParticipant.draw_position.is_not(None))
|
|
.order_by(RoundParticipant.draw_position)
|
|
)
|
|
).all()
|
|
users = {
|
|
user.id: user
|
|
for user in (
|
|
await session.scalars(select(User).where(User.id.in_({p.user_id for p in participants})))
|
|
).all()
|
|
}
|
|
ordered_rows = [(p.bet_txid, p.bet_vout, users[p.user_id].address) for p in participants]
|
|
if participants_hash(ordered_rows) != committed_hash:
|
|
raise RuntimeError(f"round {round_id}: the draw list no longer matches its published fingerprint")
|
|
seed = draw_seed(server_seed, block_hash, committed_hash)
|
|
winner = participants[draw_index(seed, len(participants))]
|
|
winner_address = users[winner.user_id].address
|
|
pool_amount = sum(p.bet_amount_sats for p in participants if p.status == "confirmed")
|
|
|
|
round_.draw_block_height = draw_height
|
|
round_.draw_block_hash = block_hash
|
|
round_.seed_int = str(seed)
|
|
round_.winner_user_id = winner.user_id
|
|
round_.pool_amount_sats = pool_amount
|
|
# The reveal (RND-18, DAT-64): the plaintext secret becomes visible in the same
|
|
# transaction that stores the winner, never before.
|
|
round_.server_seed = server_seed
|
|
round_.status = "paying_out"
|
|
await write_audit_log(
|
|
session,
|
|
"winner_drawn",
|
|
{
|
|
"winner_address": winner_address,
|
|
"pool_amount_sats": pool_amount,
|
|
"block_height": draw_height,
|
|
"block_hash": block_hash,
|
|
"participants": len(participants),
|
|
"draw_height": draw_height,
|
|
"participants_hash": committed_hash,
|
|
"server_seed": server_seed,
|
|
"seed": f"{seed:064x}",
|
|
},
|
|
user_id=winner.user_id,
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
broadcaster.publish()
|
|
|
|
logger.info("round %s: winner=%s pool=%s", round_id, winner_address, pool_amount)
|
|
await self._trigger_payout(round_id)
|
|
|
|
async def _wait_for_draw_block(
|
|
self, round_id: int, draw_height: int, confirmations: int, waiting_since: datetime
|
|
) -> str:
|
|
"""The hash of the block at `draw_height`, once it is safe to seed the draw with it
|
|
(RND-15, RND-16, RND-20; B-73): the chain must have `confirmations` blocks from that
|
|
height (a tip of at least `draw_height + confirmations - 1`), and then the quorum of
|
|
servers must agree on the header *at that exact height* — never on the tip, never on
|
|
a later block. If they do not agree the same height is asked again at the next step
|
|
and `draw_header_corroboration_failed` is written once per distinct hash the active
|
|
server reported, so a disagreement that lasts does not write a row every 5 seconds.
|
|
Whoever produces or relays blocks therefore cannot choose which block seeds the draw:
|
|
withholding or delaying one only delays the draw.
|
|
|
|
The wait has no timeout — the entropy genuinely depends on a future block — but it is
|
|
observable (B-36): progress is logged every _DRAW_PROGRESS_LOG_INTERVAL_SECONDS, and a
|
|
`draw_stalled` row is written when for _DRAW_STALL_THRESHOLD_SECONDS the chain has not
|
|
advanced while the round waits for confirmations, or the quorum has been failing to
|
|
agree — and repeated every threshold for as long as that lasts. The clock restarts
|
|
whenever the tip advances during the confirmation wait, so a chain that is merely
|
|
producing its three blocks at the normal pace never counts as stalled.
|
|
"""
|
|
required_tip = draw_height + confirmations - 1
|
|
now = datetime.now(timezone.utc)
|
|
next_progress_log_at = now + timedelta(seconds=_DRAW_PROGRESS_LOG_INTERVAL_SECONDS)
|
|
stall_since = now
|
|
next_stall_audit_at = now + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
|
last_tip = self._listener.tip_height
|
|
agreeing = False # True once the confirmations are there and only the quorum is awaited
|
|
reported_hashes: set[str | None] = set()
|
|
|
|
while True:
|
|
self.last_heartbeat = time.monotonic()
|
|
now = datetime.now(timezone.utc)
|
|
tip = self._listener.tip_height
|
|
if tip >= required_tip:
|
|
if not agreeing:
|
|
agreeing = True
|
|
stall_since = now
|
|
next_stall_audit_at = now + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
|
block_hash, reported = await self._listener.agreed_block_hash(draw_height)
|
|
if block_hash is not None:
|
|
return block_hash
|
|
if reported not in reported_hashes:
|
|
reported_hashes.add(reported)
|
|
logger.error(
|
|
"round %s: block %s header (active server: %s) is not agreed by the other Electrum "
|
|
"servers; asking again for the same height",
|
|
round_id,
|
|
draw_height,
|
|
reported,
|
|
)
|
|
async with self._session_factory() as session:
|
|
await write_audit_log(
|
|
session,
|
|
"draw_header_corroboration_failed",
|
|
{"height": draw_height, "reported_hash": reported},
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
elif tip != last_tip:
|
|
last_tip = tip
|
|
stall_since = now
|
|
next_stall_audit_at = now + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
|
|
|
if now >= next_progress_log_at:
|
|
logger.info(
|
|
"round %s: still waiting for block %s (%s confirmation(s) needed, chain at %s, %.0fs since drawing started)",
|
|
round_id,
|
|
draw_height,
|
|
confirmations,
|
|
tip,
|
|
(now - waiting_since).total_seconds(),
|
|
)
|
|
next_progress_log_at = now + timedelta(seconds=_DRAW_PROGRESS_LOG_INTERVAL_SECONDS)
|
|
if now >= next_stall_audit_at:
|
|
async with self._session_factory() as session:
|
|
await write_audit_log(
|
|
session,
|
|
"draw_stalled",
|
|
{
|
|
"draw_height": draw_height,
|
|
"current_tip_height": tip,
|
|
"elapsed_seconds": int((now - stall_since).total_seconds()),
|
|
},
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
next_stall_audit_at = now + timedelta(seconds=_DRAW_STALL_THRESHOLD_SECONDS)
|
|
await asyncio.sleep(_TICK_INTERVAL_SECONDS)
|
|
|
|
async def _retry_payout_if_due(self, round_id: int) -> None:
|
|
"""B-26: whether a "paying_out" round is due for another payout attempt.
|
|
|
|
Throttled by the most recent payout_failed audit entry for this round
|
|
(written by _log_payout_failure on every early return in _trigger_payout,
|
|
including ones that used to fail silently) rather than by any new DB state,
|
|
since a failed attempt doesn't necessarily leave a PendingTransaction behind
|
|
(a build failure like a missing fee_address never gets that far). No entry
|
|
yet means this round hasn't failed before — either it's a fresh "paying_out"
|
|
(the very first call already happened from _close_and_draw and hasn't had a
|
|
chance to fail yet) or the process restarted before ever recording one —
|
|
either way it's due immediately.
|
|
"""
|
|
async with self._session_factory() as session:
|
|
if await session.scalar(select(Round.payout_blocked_at).where(Round.id == round_id)) is not None:
|
|
return # B-82 (DAT-68): held until an operator resumes the lottery
|
|
last_failure_at = await session.scalar(
|
|
select(AuditLog.created_at)
|
|
.where(AuditLog.event_type == "payout_failed", AuditLog.round_id == round_id)
|
|
.order_by(AuditLog.id.desc())
|
|
.limit(1)
|
|
)
|
|
if last_failure_at is not None:
|
|
last_failure_at = last_failure_at.replace(tzinfo=timezone.utc)
|
|
if datetime.now(timezone.utc) < last_failure_at + timedelta(seconds=_PAYOUT_RETRY_INTERVAL_SECONDS):
|
|
return # too soon — avoid hammering a persistently-broken payout
|
|
await self._trigger_payout(round_id)
|
|
|
|
async def _trigger_payout(self, round_id: int) -> None:
|
|
"""Four phases, so no DB session is held across a network call (B-18): read
|
|
what's needed, build the tx, persist the intent, then broadcast.
|
|
|
|
The persist happens *before* the broadcast (B-25) — the same two-phase shape
|
|
as place_bet/request_withdrawal (B-08): a crash between building the payout
|
|
and recording it used to leave money on-chain with zero trace in the DB (no
|
|
payout_txid, no PendingTransaction), so a manual retry would have paid the
|
|
winner a second time. Now the worst case is a "building" PendingTransaction
|
|
the reconciler (app/tx/reconcile.py) can resolve either way by asking the
|
|
chain whether the tx exists, exactly like it already does for bets and
|
|
withdrawals.
|
|
"""
|
|
client = self._listener.client
|
|
if client is None:
|
|
logger.error("round %s payout deferred: not connected", round_id)
|
|
await self._log_payout_failure(round_id, None, "electrum client not connected")
|
|
return
|
|
|
|
# --- Phase 1: read (session closed before any network I/O) ---------------
|
|
async with self._session_factory() as session:
|
|
try:
|
|
await begin_immediate(session) # B-88
|
|
except OperationalError:
|
|
# Nothing built or sent yet: try again at the next tick.
|
|
logger.warning("round %s payout deferred: database busy", round_id)
|
|
return
|
|
round_ = await session.get(Round, round_id)
|
|
if round_.payout_blocked_at is not None:
|
|
# B-82 (DAT-68): held since the winner's own bet left the chain. Nothing is built
|
|
# until an operator resumes the lottery, which clears the mark (and the next
|
|
# attempt verifies the coins again).
|
|
return
|
|
already_in_flight = await session.scalar(
|
|
select(PendingTransaction).where(
|
|
PendingTransaction.round_id == round_id,
|
|
PendingTransaction.kind == "payout",
|
|
PendingTransaction.status.in_(("building", "pending")),
|
|
)
|
|
)
|
|
if already_in_flight is not None:
|
|
# A payout for this round is already building or broadcast — this
|
|
# must not build a second one, or a retry (manual, or a future
|
|
# automatic one) would pay the winner twice. Confirmation/
|
|
# reconciliation already owns resolving that row.
|
|
logger.info(
|
|
"round %s payout already in flight (pending_transaction %s), skipping",
|
|
round_id,
|
|
already_in_flight.id,
|
|
)
|
|
return
|
|
reserved_outpoints = await _reserved_payout_outpoints(session)
|
|
config = await get_round_config(session)
|
|
fee_address = round_.fee_address # B-95 (DAT-61): the round's own, fixed when it opened
|
|
fee_rate = config.fee_rate_sat_vb
|
|
pool_amount_sats = round_.pool_amount_sats
|
|
round_pool_address = round_.pool_address # B-80 ("" fails in derive_round_pool_key below)
|
|
winner_user_id = round_.winner_user_id
|
|
winner = await session.get(User, winner_user_id)
|
|
winner_address = winner.address if winner is not None else None
|
|
# B-82 (RND-22, RND-39): the bets the payout may spend and must first verify — the
|
|
# round's confirmed participants, each with the output that pays the pool.
|
|
bet_rows = [
|
|
(p.id, p.user_id, p.bet_txid, p.bet_vout, p.bet_amount_sats)
|
|
for p in (
|
|
await session.scalars(
|
|
select(RoundParticipant)
|
|
.where(RoundParticipant.round_id == round_id, RoundParticipant.status == "confirmed")
|
|
.order_by(RoundParticipant.draw_position)
|
|
)
|
|
).all()
|
|
]
|
|
await session.commit() # get_round_config may have created the row
|
|
|
|
if not fee_address:
|
|
logger.error(
|
|
"round %s payout blocked: no fee_address configured (set it via the admin endpoint)", round_id
|
|
)
|
|
await self._log_payout_failure(round_id, winner_user_id, "no fee_address configured")
|
|
return
|
|
if winner_address is None:
|
|
logger.error("round %s payout blocked: winner user %s not found", round_id, winner_user_id)
|
|
await self._log_payout_failure(round_id, winner_user_id, "winner user not found")
|
|
return
|
|
|
|
# A round opened before the per-round pool address (B-80) pays from the shared reserve
|
|
# address, whose coins are not the bets of one round: it keeps the old way of choosing
|
|
# them. Every other round is verified and paid from its own bets (RND-22, RND-39).
|
|
legacy_pool = round_pool_address == derive_pool_address()
|
|
|
|
# --- Phase 2: build (network read only, no DB write yet) -----------------
|
|
try:
|
|
# B-80 (RND-22): the round's own key and address — only the coins at this
|
|
# round's pool address are ever read, so no other round's money (and no
|
|
# stray coin sent to another address) can end up among the inputs.
|
|
pool_key = derive_round_pool_key(round_id, round_pool_address)
|
|
pool_script_obj = script.p2wpkh(pool_key.to_public())
|
|
pool_address = pool_script_obj.address(network=PLM_MAINNET)
|
|
if not legacy_pool:
|
|
# B-82 (RND-39): with the quorum, before anything is built — bets that left the
|
|
# chain drop out and the split is computed on what is really there.
|
|
verified = await self._verify_round_pool(
|
|
round_id, pool_address, winner_user_id, bet_rows, pool_amount_sats
|
|
)
|
|
if verified is None:
|
|
return # held, unverifiable or database busy — already recorded
|
|
bet_rows, pool_amount_sats = verified
|
|
winner_sats = winner_share(pool_amount_sats)
|
|
commission_share = pool_amount_sats - winner_sats # remainder from rounding goes to fees
|
|
if legacy_pool:
|
|
entries = await client.listunspent(address_to_scripthash(pool_address))
|
|
utxos = [
|
|
Utxo(e["tx_hash"], e["tx_pos"], e["value"])
|
|
for e in entries
|
|
if e["height"] > 0 and (e["tx_hash"], e["tx_pos"]) not in reserved_outpoints
|
|
]
|
|
else:
|
|
# B-82 (RND-22): exactly the verified bet outputs of this round, nothing else
|
|
# that happens to sit at the address, and none already claimed by an unresolved
|
|
# payout.
|
|
utxos = [
|
|
Utxo(txid, vout, amount)
|
|
for _id, _user_id, txid, vout, amount in bet_rows
|
|
if (txid, vout) not in reserved_outpoints
|
|
]
|
|
|
|
built = build_payout_transaction(
|
|
signing_key=pool_key,
|
|
from_script=pool_script_obj,
|
|
utxos=utxos,
|
|
winner_address=winner_address,
|
|
winner_share_sats=winner_sats,
|
|
fee_address=fee_address,
|
|
commission_sats=commission_share,
|
|
reserve_address=derive_pool_address(), # B-79 (RND-38): the bump reserve's destination
|
|
fee_rate_sat_vb=fee_rate,
|
|
)
|
|
except InsufficientFundsError as exc:
|
|
# Includes the "too_many_inputs" case: the pool holds enough, but spread over
|
|
# more UTXOs than one transaction may spend, so /admin has to say which. Since
|
|
# B-52 that means MAX_PAYOUT_TX_INPUTS, and participants are capped below it at
|
|
# bet time (bets/service.py), so reaching it now takes pool change accumulated
|
|
# over many rounds rather than one busy round — an operator consolidation job,
|
|
# not a dead end for the bets of the round in progress.
|
|
reason = "insufficient pool UTXOs" if exc.code == "insufficient_balance" else exc.code
|
|
logger.exception("round %s payout failed: %s", round_id, reason)
|
|
await self._log_payout_failure(round_id, winner_user_id, reason)
|
|
return
|
|
except Exception:
|
|
# Anything else — a malformed fee_address (EmbitError) or similar. This
|
|
# used to escape all the way to run()'s catch-all, which logged it
|
|
# without recording anything, leaving no trace of *why* the round was
|
|
# stuck (B-05). _retry_payout_if_due (B-26) is what turns this recorded
|
|
# failure into an automatic retry instead of a dead end.
|
|
logger.exception("round %s payout build failed", round_id)
|
|
await self._log_payout_failure(round_id, winner_user_id, "payout build failed")
|
|
return
|
|
|
|
# --- Phase 3: persist the intent, *then* broadcast (B-25) -----------------
|
|
async with self._session_factory() as session:
|
|
try:
|
|
await begin_immediate(session) # B-88
|
|
except OperationalError:
|
|
# Not written, not broadcast: the payout is simply rebuilt on the retry.
|
|
logger.warning("round %s payout deferred before broadcast: database busy", round_id)
|
|
await self._log_payout_failure(round_id, winner_user_id, "database busy")
|
|
return
|
|
round_ = await session.get(Round, round_id)
|
|
round_.winner_amount_sats = built.winner_sats
|
|
round_.fee_amount_sats = built.commission_sats
|
|
round_.bump_reserve_sats = built.reserve_sats # B-79 (DAT-73, DAT-22)
|
|
round_.payout_txid = built.txid
|
|
pending = PendingTransaction(
|
|
kind="payout",
|
|
round_id=round_id,
|
|
current_txid=built.txid,
|
|
fee_rate_sat_vb=fee_rate,
|
|
raw_tx_hex=built.raw_hex,
|
|
# "building" until the broadcast succeeds, exactly like place_bet's
|
|
# two phases — see the reconciler, which gives this a short grace
|
|
# period before asking the chain whether it made it out after all.
|
|
status="building",
|
|
)
|
|
session.add(pending)
|
|
await record_version(session, pending, built.txid, built.raw_hex) # B-77: the first version
|
|
await session.commit()
|
|
pending_id = pending.id
|
|
|
|
# --- Phase 4: broadcast, then promote the pending row --------------------
|
|
# B-76 (TX-05): the same three outcomes as a bet's broadcast. Accepted — the
|
|
# server took it, or refused with "already known" / a server of the network
|
|
# holds it — promotes the row; a refusal and an uncertain outcome both leave it
|
|
# "building" with a payout_failed entry, since which of the two it really was
|
|
# is the reconciler's call, made against the chain (TX-16, TX-17).
|
|
verdict = await broadcast_with_verdict(client, self._listener, built.raw_hex, built.txid)
|
|
if verdict.outcome is not BroadcastOutcome.ACCEPTED:
|
|
logger.error("round %s payout broadcast not accepted (%s): %s", round_id, verdict.outcome.value, verdict.reason)
|
|
await self._log_payout_failure(round_id, winner_user_id, "broadcast rejected")
|
|
return
|
|
|
|
async with self._session_factory() as session:
|
|
try:
|
|
await begin_immediate(session) # B-88
|
|
except OperationalError:
|
|
# The tx is on the network; the row stays "building" and the
|
|
# reconciler finds it on the chain and promotes it (TX-28).
|
|
logger.error(
|
|
"round %s payout %s broadcast but the database is busy: left 'building'",
|
|
round_id,
|
|
built.txid,
|
|
)
|
|
return
|
|
pending = await session.get(PendingTransaction, pending_id)
|
|
pending.status = "pending"
|
|
await write_audit_log(
|
|
session,
|
|
"payout_sent",
|
|
{"txid": built.txid, "winner_sats": built.winner_sats, "commission_sats": built.commission_sats},
|
|
user_id=winner_user_id,
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
|
|
logger.info("round %s payout broadcast: txid=%s", round_id, built.txid)
|
|
|
|
async def _verify_round_pool(
|
|
self,
|
|
round_id: int,
|
|
pool_address: str,
|
|
winner_user_id: int | None,
|
|
bet_rows: list[tuple[int, int, str, int, int]],
|
|
pool_amount_sats: int | None,
|
|
) -> tuple[list[tuple[int, int, str, int, int]], int] | None:
|
|
"""B-82 (RND-39, DAT-67, DAT-68, ELC-27): before a payout is built, check with the quorum
|
|
that each confirmed participant's bet output is still at the round's pool address.
|
|
|
|
Returns the participants that are really there with the pool amount recomputed on them
|
|
(the same pair when nothing was missing), or None when this attempt must stop: the
|
|
quorum could not be reached (`payout_failed`, `pool verification unavailable`), the
|
|
database was busy, or the **winner's own bet is gone** — then the payout is held
|
|
(`payout_blocked`, the lottery pauses) and only an operator's resume tries again.
|
|
|
|
Participants whose bet is gone become `reorged_out` and leave the pool, one
|
|
`participant_reorged_out` row each, in one transaction with the rewritten
|
|
`pool_amount_sats` (which can only go down). The winner's own row is left `confirmed`
|
|
in the held case, precisely so that the verification repeats after the resume."""
|
|
verdicts = await self._listener.verify_pool_outpoints(
|
|
address_to_scripthash(pool_address), [(txid, vout, amount) for _i, _u, txid, vout, amount in bet_rows]
|
|
)
|
|
if verdicts is None:
|
|
logger.error("round %s payout deferred: the pool coins could not be verified with the quorum", round_id)
|
|
await self._log_payout_failure(round_id, winner_user_id, "pool verification unavailable")
|
|
return None
|
|
gone = [row for row in bet_rows if not verdicts[(row[2], row[3])]]
|
|
if not gone:
|
|
return bet_rows, pool_amount_sats
|
|
|
|
winner_gone = next((row for row in gone if row[1] == winner_user_id), None)
|
|
async with self._session_factory() as session:
|
|
try:
|
|
await begin_immediate(session) # B-88
|
|
except OperationalError:
|
|
logger.warning("round %s payout deferred: database busy while excluding reorged-out bets", round_id)
|
|
return None
|
|
round_ = await session.get(Round, round_id)
|
|
excluded = [row for row in gone if row is not winner_gone]
|
|
for participant_id, user_id, txid, vout, amount in excluded:
|
|
participant = await session.get(RoundParticipant, participant_id)
|
|
if participant is None or participant.status != "confirmed":
|
|
continue
|
|
participant.status = "reorged_out"
|
|
await session.flush()
|
|
remaining = (
|
|
await session.scalars(
|
|
select(RoundParticipant).where(
|
|
RoundParticipant.round_id == round_id, RoundParticipant.status == "confirmed"
|
|
)
|
|
)
|
|
).all()
|
|
new_pool = sum(p.bet_amount_sats for p in remaining)
|
|
if pool_amount_sats is not None:
|
|
new_pool = min(new_pool, pool_amount_sats) # DAT-67: only ever reduced
|
|
round_.pool_amount_sats = new_pool
|
|
for _participant_id, user_id, txid, vout, amount in excluded:
|
|
await write_audit_log(
|
|
session,
|
|
"participant_reorged_out",
|
|
{"txid": txid, "vout": vout, "amount_sats": amount, "pool_amount_sats": new_pool},
|
|
user_id=user_id,
|
|
round_id=round_id,
|
|
)
|
|
if winner_gone is not None:
|
|
_pid, user_id, txid, vout, amount = winner_gone
|
|
round_.payout_blocked_at = datetime.now(timezone.utc)
|
|
await write_audit_log(
|
|
session,
|
|
"payout_blocked",
|
|
{"txid": txid, "vout": vout, "amount_sats": amount},
|
|
user_id=user_id,
|
|
round_id=round_id,
|
|
)
|
|
logger.error(
|
|
"round %s payout held: the winner's bet %s:%s is no longer on the chain; lottery paused",
|
|
round_id,
|
|
txid,
|
|
vout,
|
|
)
|
|
await pause_automatically(session, "payout_blocked") # same transaction (AUD-44)
|
|
await session.commit()
|
|
broadcaster.publish()
|
|
if winner_gone is not None:
|
|
return None
|
|
gone_ids = {row[0] for row in gone}
|
|
return [row for row in bet_rows if row[0] not in gone_ids], new_pool
|
|
|
|
async def _log_payout_failure(self, round_id: int, winner_user_id: int | None, reason: str) -> None:
|
|
"""Leaves an operator-visible trace in the audit log for a round stuck in
|
|
"paying_out" — the logs alone don't show up in /admin. Called from every
|
|
early return in _trigger_payout (B-26), not just the generic exception
|
|
branch as before, so _retry_payout_if_due always has an entry to throttle
|
|
against and /admin always shows *why* a round is stuck rather than just
|
|
that it is."""
|
|
try:
|
|
async with self._session_factory() as session:
|
|
await write_audit_log(
|
|
session,
|
|
"payout_failed",
|
|
{"round_id": round_id, "reason": reason},
|
|
user_id=winner_user_id,
|
|
round_id=round_id,
|
|
)
|
|
await session.commit()
|
|
except Exception:
|
|
logger.exception("could not record the payout failure of round %s", round_id)
|
|
|
|
|
|
async def _reserved_payout_outpoints(session: AsyncSession) -> set[tuple[str, int]]:
|
|
"""Pool UTXOs already claimed by a payout that hasn't resolved yet — this
|
|
round's own in-flight payout (guarded against separately in _trigger_payout) or
|
|
a stale one from an earlier round the reconciler hasn't abandoned yet (B-25).
|
|
These must be excluded from selection, or a retry would double-spend the same
|
|
coins into two payouts before the reconciler gets a chance to release them."""
|
|
rows = (
|
|
await session.scalars(
|
|
select(PendingTransaction).where(
|
|
PendingTransaction.kind == "payout",
|
|
PendingTransaction.status.in_(("building", "pending")),
|
|
)
|
|
)
|
|
).all()
|
|
reserved: set[tuple[str, int]] = set()
|
|
for row in rows:
|
|
tx = Transaction.parse(bytes.fromhex(row.raw_tx_hex))
|
|
for vin in tx.vin:
|
|
reserved.add((vin.txid.hex(), vin.vout))
|
|
return reserved
|