Abandon a transaction only on proof, and keep watching failed rows (B-78, TX-15, TX-17, TX-23, TX-26, ELC-28, DAT-33, DAT-36, DAT-69, DAT-72, AUD-11, AUD-12, AUD-31, AUD-39, API-32)

reconcile_once abandoned a row when the *active* server's history lacked its txid:
after 120s for `building` (which a restart onto a different server makes likely),
after 6h for `pending`, and all at once after a forward clock jump. It re-checked the
status but not the txid, so it raced the bumper, and a `failed` row was never looked at
again.

- The reconciler asks every server (ElectrumListener.get_histories, active + others) for
  the history of the row's address and looks for every txid of the row. Present on any
  server -> promoted (adopting the version found). Absent is only "no server of the
  quorum knows it" and needs the active server plus the quorum minimum of the others to
  have answered; even then nothing is released without proof:
  (a) an input spent in a block by a transaction that is none of the row's versions,
      agreed by a strict majority of a quorum of the other servers (confirmed_spender), or
  (b) a rebroadcast of the latest raw tx to every server (rebroadcast_everywhere)
      refused by all of them, explicitly, for missing/spent inputs.
  Accepted or "already known" by any server means alive; a fee refusal is not proof
  (`building` -> `pending` for the bumper, `pending` untouched); too few answers or
  anything else leaves the row alone. failure_reason names the proof.
- The write re-reads status *and* current_txid under begin_immediate (TX-26).
- pending_transactions.failed_at (set by _abandon and _release_failed_withdrawal) starts
  a 14-day watch: a failed row that a majority of the other servers (at least the quorum
  minimum) list in a block becomes `confirmed`, writes pending_tx_resurrected once and
  pauses the lottery with paused_reason `resurrected_tx`; linked rows are not corrected.
  The confirmation is by quorum of histories for now; merkle proofs come with B-83.
- round_config.paused_reason (migration b7d2e41c9a05, with the failed_at one): `operator`
  from POST /admin/pause (which accepts an optional `note` and never overwrites an
  automatic reason), cleared by resume, which logs the reason it lifted; exposed on
  /admin/config and GET /rounds/current. rounds/pause.py:pause_automatically is the
  system's way in.
- PendingTransactionReconciler takes the listener as `network`; with none it can promote
  but never abandon.

Tests changed because the spec changed: the three reconciler tests that expect an
abandonment (and the one in test_tx_versions) now supply a network that proves the
transaction is gone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
davideandClaude Sonnet 5.5 committed 2026-10-14 15:07:09 +02:00
1 parent f936d6660c
commit a4b160e240
29 files changed
+1280 -201

No files matched your search

+26 -4
View File
@@ -13,6 +13,7 @@ from app.config import settings
from app.db.models import AuditLog, BugReport, PendingTransaction, Round, User
from app.db.session import get_session
from app.rounds.config import get_round_config
from app.rounds.pause import AUTOMATIC_REASONS, OPERATOR_REASON
from app.wallet.address import is_valid_plm_address
from app.wallet.hd import derive_user_wif
from app.wallet.psbt_builder import MAX_FEE_RATE_SAT_VB
@@ -56,6 +57,7 @@ class RoundConfigResponse(BaseModel):
rbf_timeout_seconds: int
draw_animation_seconds: int
paused: bool
paused_reason: str | None = None # B-78 (DAT-69): why, or None when not paused
class RoundConfigUpdate(BaseModel):
@@ -93,6 +95,7 @@ class RoundConfigUpdate(BaseModel):
def _config_response(config) -> RoundConfigResponse:
fields = {field: getattr(config, field) for field in _CONFIG_FIELDS}
fields["paused"] = config.paused
fields["paused_reason"] = config.paused_reason
return RoundConfigResponse(**fields)
@@ -128,14 +131,29 @@ async def update_config(
return _config_response(config)
class PauseRequest(BaseModel):
note: str | None = Field(default=None, max_length=256)
@router.post("/pause", response_model=RoundConfigResponse, dependencies=[Depends(require_admin)])
async def pause_lottery(session: AsyncSession = Depends(get_session)) -> RoundConfigResponse:
async def pause_lottery(
body: PauseRequest | None = None, session: AsyncSession = Depends(get_session)
) -> RoundConfigResponse:
"""Maintenance switch: the round in progress (if any) still closes, draws,
and pays out its winner normally — only opening the *next* round is
suppressed until /admin/resume is called (rounds/service.py)."""
suppressed until /admin/resume is called (rounds/service.py).
B-78 (DAT-69, API-32): the reason is "operator", except that an automatic reason
already in force stays — it is the one that needs a look. The audit row always
says `operator` for this call."""
config = await get_round_config(session)
if not (config.paused and config.paused_reason in AUTOMATIC_REASONS):
config.paused_reason = OPERATOR_REASON
config.paused = True
await write_audit_log(session, "lottery_paused", {})
payload = {"reason": OPERATOR_REASON}
if body is not None and body.note:
payload["note"] = body.note
await write_audit_log(session, "lottery_paused", payload)
await session.commit()
return _config_response(config)
@@ -143,8 +161,12 @@ async def pause_lottery(session: AsyncSession = Depends(get_session)) -> RoundCo
@router.post("/resume", response_model=RoundConfigResponse, dependencies=[Depends(require_admin)])
async def resume_lottery(session: AsyncSession = Depends(get_session)) -> RoundConfigResponse:
config = await get_round_config(session)
# B-78 (AUD-12): the reason the resume took away, so the log says what the operator
# agreed to resume (None when the lottery was not paused).
lifted = config.paused_reason if config.paused else None
config.paused = False
await write_audit_log(session, "lottery_resumed", {})
config.paused_reason = None
await write_audit_log(session, "lottery_resumed", {"reason": lifted})
await session.commit()
return _config_response(config)
+3
View File
@@ -120,6 +120,7 @@ class CurrentRoundResponse(BaseModel):
draw_waiting_since: str | None = None
chain_tip_height: int | None = None
lottery_paused: bool = False
paused_reason: str | None = None # B-78 (DAT-69): operator / resurrected_tx / ..., None if not paused
# B-66: false while the instance is missing configuration a round cannot run
# without (today: fee_address) — no round will open until it's set, so this is
# the difference between "wait, the next round is coming" and "nothing is coming
@@ -147,6 +148,7 @@ async def current_round(
draw_animation_seconds=config.draw_animation_seconds,
chain_tip_height=chain_tip_height,
lottery_paused=config.paused,
paused_reason=config.paused_reason,
lottery_configured=rounds_can_open(config),
)
@@ -225,6 +227,7 @@ async def current_round(
draw_waiting_since=isoformat_utc(round_.drawing_started_at) if round_.status == "drawing" else None,
chain_tip_height=chain_tip_height,
lottery_paused=config.paused,
paused_reason=config.paused_reason,
lottery_configured=rounds_can_open(config),
user_played=user_played,
)
+9
View File
@@ -155,6 +155,12 @@ class RoundConfig(Base):
# completion (closes, draws, pays out the winner) but no new round is opened
# afterwards — see rounds/service.py:open_new_round_if_needed.
paused: Mapped[bool] = mapped_column(default=False)
# B-78 (DAT-69): why the lottery is paused — NULL exactly when `paused` is false.
# "operator" (POST /admin/pause) or an automatic reason the system wrote itself
# ("resurrected_tx" so far; "reorg" and "payout_blocked" belong to B-85 and B-82).
# An automatic reason is never overwritten by an operator pause, so the cause that
# needs a look is not lost; POST /admin/resume clears whichever it is.
paused_reason: Mapped[str | None] = mapped_column(String(16), default=None)
updated_at: Mapped[datetime] = mapped_column(default=utcnow, onupdate=utcnow)
@@ -197,6 +203,9 @@ class PendingTransaction(Base):
attempt_count: Mapped[int] = mapped_column(default=1)
# Why the reconciler gave up on this tx — operator-facing, only set on "failed".
failure_reason: Mapped[str | None] = mapped_column(String(128), default=None)
# B-78 (DAT-72): when the row went to "failed". A failed row stays watched for
# FAILED_WATCH_DAYS from here: a transaction given up for dead can still confirm.
failed_at: Mapped[datetime | None] = mapped_column(default=None)
updated_at: Mapped[datetime] = mapped_column(default=utcnow, onupdate=utcnow)
+74
View File
@@ -1,5 +1,6 @@
import asyncio
import logging
from collections import Counter
from collections.abc import Callable
from embit.transaction import Transaction
@@ -16,6 +17,7 @@ from app.deposits.service import (
)
from app.config import MIN_QUORUM_RESPONSES
from app.electrum.client import ElectrumClient, ElectrumEndpoint, ElectrumRpcError
from app.electrum.quorum import ServerAnswers
from app.electrum.scripthash import address_to_scripthash
from app.rounds.draw import (
HeaderValidationError,
@@ -497,6 +499,78 @@ class ElectrumListener:
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
# --- What the network knows of a transaction (B-78, TX-15, TX-17) -------------------
async def _ask_everyone(self, ask: Callable[[ElectrumClient], "asyncio.Future"]) -> ServerAnswers:
active, others = await asyncio.gather(self._ask_active_server(ask), self._ask_other_servers(ask))
return ServerAnswers(active, tuple(others), self._minimum_other_answers())
async def get_histories(self, scripthash: str) -> ServerAnswers:
"""`blockchain.scripthash.get_history` of `scripthash` from the active server and
every other one: each answer is the list of {"tx_hash", "height"} entries."""
async def _ask(client: ElectrumClient) -> list | None:
try:
history = await client.get_history(scripthash)
except Exception:
return None
return history if isinstance(history, list) else None
return await self._ask_everyone(_ask)
async def rebroadcast_everywhere(self, raw_tx_hex: str) -> ServerAnswers:
"""Send `raw_tx_hex` to the active server and every other one and report what each
said: ("accepted", "") or ("refused", <its message>) — an explicit JSON-RPC error —
and None for a server that gave no answer at all (TX-17). A tx the servers already
have comes back as a refusal with an "already known" message; the caller reads it."""
async def _ask(client: ElectrumClient) -> tuple | None:
try:
await client.broadcast(raw_tx_hex)
except ElectrumRpcError as exc:
return ("refused", exc.message)
except Exception:
return None
return ("accepted", "")
return await self._ask_everyone(_ask)
async def confirmed_spender(
self, scripthash: str, own_txids: set[str], outpoints: set[tuple[str, int]]
) -> str | None:
"""The txid of a transaction, none of the row's own versions, that spends one of
`outpoints` and is **in a block** — the first proof by which a row may be given
up (TX-17): its inputs went elsewhere. Asked of the *other* servers only (the
active one is what a hostile peer would control) and returned only when at least
the quorum minimum answered and a strict majority of the answerers name the same
txid. Each server looks through the confirmed history of the address the row
spends from.
"""
async def _ask(client: ElectrumClient) -> str | None:
history = await client.get_history(scripthash)
for entry in history:
txid = entry.get("tx_hash")
if txid in own_txids or (entry.get("height") or 0) <= 0:
continue
raw = await client.get_transaction(txid, False)
tx = Transaction.parse(bytes.fromhex(raw))
if tx.txid().hex() != txid:
continue
if any((vin.txid.hex(), vin.vout) in outpoints for vin in tx.vin):
return txid
return "" # looked everywhere: no conflicting transaction
answers = await self._ask_other_servers(_ask)
answered = [answer for answer in answers if answer is not None]
if len(answered) < max(self._minimum_other_answers(), 1):
return None
counts = Counter(answer for answer in answered if answer)
if not counts:
return None
txid, votes = counts.most_common(1)[0]
return txid if votes * 2 > len(answered) else None
async def corroborate_header(self, height: int, expected_hash: str) -> bool:
"""B-28: is `expected_hash` — the header our own active connection
reported for `height` — corroborated by other configured servers before
+27
View File
@@ -0,0 +1,27 @@
"""What a question put to every server came back with (B-78).
`active` is the answer of the connection the platform is running on, `others` those of
each other configured server in configuration order; `None` stands for "no usable
answer" (unreachable, timed out, an error that is not an answer)."""
from typing import NamedTuple
class ServerAnswers(NamedTuple):
active: object | None
others: tuple
others_needed: int # how many of `others` must have answered for the network to count
@property
def answered_others(self) -> list:
return [answer for answer in self.others if answer is not None]
@property
def complete(self) -> bool:
"""Enough of the network answered for its silence to mean something: the active
server and at least `others_needed` of the others."""
return self.active is not None and len(self.answered_others) >= self.others_needed
@property
def all(self) -> list:
return ([] if self.active is None else [self.active]) + self.answered_others
+1 -1
View File
@@ -66,7 +66,7 @@ async def lifespan(app: FastAPI):
bumper = RbfBumper(AsyncSessionLocal, lambda: listener.client, network=listener)
# Resolves in-flight transactions against the chain — the piece that lets the
# system recover on its own from a broadcast that never confirmed (B-04/B-08).
reconciler = PendingTransactionReconciler(AsyncSessionLocal, lambda: listener.client)
reconciler = PendingTransactionReconciler(AsyncSessionLocal, lambda: listener.client, network=listener)
# Periodic safety net for deposit crediting/external-spend detection,
# independent of scripthash-change notifications — catches a subscription
# silently lost on an otherwise healthy connection (B-30).
+30
View File
@@ -0,0 +1,30 @@
"""Pausing the lottery from code (B-78, DAT-69, AUD-11).
A pause only stops the *next* round from opening (rounds/service.py:rounds_can_open); a
round in progress still closes, draws and pays. Besides the operator's own switch there
are pauses the system raises itself when something needs a human look — a transaction
given up for dead that confirmed after all (`resurrected_tx`, B-78) and, in later
findings, a deep reorg (`reorg`, B-85) and a payout whose winning bet left the chain
(`payout_blocked`, B-82).
"""
from sqlalchemy.ext.asyncio import AsyncSession
from app.audit.log import write_audit_log
from app.rounds.config import get_round_config
OPERATOR_REASON = "operator"
AUTOMATIC_REASONS = ("reorg", "payout_blocked", "resurrected_tx")
async def pause_automatically(session: AsyncSession, reason: str) -> None:
"""Pause the lottery for an automatic `reason` in the caller's transaction and write
the `lottery_paused` row. An automatic reason is written even over a pause that is
already active (DAT-69): the cause that needs checking must not be hidden behind an
earlier one, and an operator pause is never what overrides it."""
if reason not in AUTOMATIC_REASONS:
raise ValueError(f"not an automatic pause reason: {reason!r}")
config = await get_round_config(session)
config.paused = True
config.paused_reason = reason
await write_audit_log(session, "lottery_paused", {"reason": reason})
+244 -48
View File
@@ -13,15 +13,26 @@ balance simply lost them, with no path back short of editing the DB by hand
What it does, per PendingTransaction that isn't already terminal:
* status "building" — we crashed (or were killed) between writing the row and
broadcasting. Ask the chain: if the tx is there after all, promote everything
to its live state; if it isn't, release the UTXOs and undo the intent.
broadcasting. Ask the network: if any server knows one of the row's txids,
promote everything to its live state.
* status "pending" — broadcast, still unconfirmed. Left alone until it has been
unconfirmed for `_ABANDON_AFTER_SECONDS`, since absence from one server's
mempool is not proof of death; only then is it abandoned like the above.
unconfirmed for `_ABANDON_AFTER_SECONDS`, and even then only examined.
Deliberately conservative: it never touches a tx the chain knows about, and the
grace period is long (multiples of the RBF timeout) so a slow-but-alive tx is
bumped by RbfBumper rather than abandoned here.
and then *nothing is given up without proof* (B-78, ADR-51, TX-17). A row is
abandoned — its coins released — only when
(a) one of its inputs is spent **in a block** by a transaction that is none of its
own versions, and the other servers agree; or
(b) no server of the quorum knows any txid of the row **and** sending its latest
raw transaction to every server is refused, explicitly and by all of them, for
missing or already-spent inputs.
Absence from one server's history proves nothing — after a restart onto a different
server that is the normal state of a tx the new server has not seen — and neither
does the age of the row. A refusal for fee reasons is no proof either: such a row is
promoted (building) or left to the RBF bumper (pending). A transport failure never
resolves anything. Failed rows stay watched for `FAILED_WATCH_DAYS`: if one of them
confirms after all, the lottery is paused for the operator (TX-23).
"""
import asyncio
@@ -35,11 +46,21 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from app.audit.log import write_audit_log
from app.db.base import begin_immediate
from app.db.models import PendingTransaction, Round, RoundParticipant, UtxoEvent, Withdrawal
from app.db.models import (
PendingTransaction,
PendingTransactionTxid,
Round,
RoundParticipant,
UtxoEvent,
Withdrawal,
)
from app.electrum.client import ElectrumClient
from app.electrum.quorum import ServerAnswers
from app.electrum.scripthash import address_to_scripthash
from app.rounds.events import broadcaster
from app.rounds.pause import pause_automatically
from app.tx.broadcast import adopt_version
from app.tx.outcome import is_already_known
from app.tx.pending_address import own_address_for
from app.tx.versions import txids_by_row, txids_of
from app.wallet.balance import recompute_balance
@@ -53,14 +74,29 @@ _POLL_INTERVAL_SECONDS = 120
_BUILDING_GRACE_SECONDS = 120
# A "pending" row was accepted by a node once. Give it a wide margin — the RBF
# bumper gets several attempts inside this window — before concluding it's gone.
# bumper gets several attempts inside this window — before it is even examined.
_ABANDON_AFTER_SECONDS = 6 * 60 * 60
# How long a failed row keeps being looked at (B-78, DAT-72): a transaction given up
# for dead can still arrive in a block.
FAILED_WATCH_DAYS = 14
# What a node says when the transaction's inputs do not exist or are spent already.
_MISSING_INPUTS_MARKERS = ("missing-inputs", "missingorspent", "missing inputs", "inputs-missing")
# What it says when only the fee is the problem: not proof of anything.
_FEE_MARKERS = ("min relay fee", "mempool min fee", "insufficient fee", "fee not met", "fee too low", "fee rate")
class PendingTransactionReconciler:
def __init__(self, session_factory: async_sessionmaker, get_client: Callable[[], ElectrumClient | None]):
def __init__(
self,
session_factory: async_sessionmaker,
get_client: Callable[[], ElectrumClient | None],
network=None,
):
self._session_factory = session_factory
self._get_client = get_client
self._network = network # the ElectrumListener: histories of every server, rebroadcast, spenders
async def run(self) -> None:
# Runs once promptly at startup: a crash mid-broadcast is exactly the case
@@ -69,7 +105,7 @@ class PendingTransactionReconciler:
client = self._get_client()
if client is not None:
try:
await reconcile_once(self._session_factory, client)
await reconcile_once(self._session_factory, client, self._network)
except asyncio.CancelledError:
raise
except Exception:
@@ -77,31 +113,41 @@ class PendingTransactionReconciler:
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
async def reconcile_once(session_factory: async_sessionmaker, client: ElectrumClient) -> int:
"""Returns how many rows were resolved (promoted or abandoned).
class _Candidate:
__slots__ = ("row_id", "status", "txid", "scripthash", "known_txids", "raw_hex")
Existence is decided by checking whether a row's own address's history
(blockchain.scripthash.get_history) includes its txid at all — mempool or
mined — rather than asking blockchain.transaction.get for a verbose reply
(B-41): several Electrum server implementations and versions reject the
verbose flag outright, and the previous substring-matching on the error
text (looking for "missing", "not found", ...) was fragile as the basis for
a decision that releases funds. A transport failure fetching history still
raises and leaves the row alone until next time — get_history not
returning our txid is the only thing that means "gone". History is cached
per scripthash within one pass, since the payout rows of one round share its
pool address (B-80).
def __init__(self, row_id, status, txid, scripthash, known_txids, raw_hex):
self.row_id = row_id
self.status = status
self.txid = txid
self.scripthash = scripthash
self.known_txids = known_txids
self.raw_hex = raw_hex
async def reconcile_once(
session_factory: async_sessionmaker, client: ElectrumClient, network=None
) -> int:
"""Returns how many rows were resolved (promoted, abandoned, realigned or
resurrected).
Existence is decided by asking every server of the quorum for the history of the
row's own address (blockchain.scripthash.get_history) and looking for **every** txid
the row has had, mempool or mined (B-41, B-77, B-78) — never for a verbose
blockchain.transaction.get reply, which several servers reject outright, and never
on error text. History is cached per scripthash within one pass, since the payout
rows of one round share its pool address (B-80). With no `network` only the active
server is asked, which can promote a row but never abandon one.
"""
now = datetime.now(timezone.utc)
async with session_factory() as session:
rows = (
await session.scalars(
select(PendingTransaction).where(PendingTransaction.status.in_(("building", "pending")))
select(PendingTransaction).where(PendingTransaction.status.in_(("building", "pending", "failed")))
)
).all()
candidates = []
# B-77 (TX-15, TX-19): a row's every txid, not only the current one.
txids_by_id = await txids_by_row(session, {row.id: row.current_txid for row in rows})
candidates: list[_Candidate] = []
for row in rows:
if not _is_due(row, now):
continue
@@ -111,56 +157,204 @@ async def reconcile_once(session_factory: async_sessionmaker, client: ElectrumCl
except Exception:
logger.exception("could not derive the address for pending_transaction %s", row.id)
continue
candidates.append((row.id, row.status, row.current_txid, scripthash, txids_by_id[row.id]))
candidates.append(
_Candidate(row.id, row.status, row.current_txid, scripthash, txids_by_id[row.id], row.raw_tx_hex)
)
resolved = 0
history_cache: dict[str, list[dict]] = {}
for row_id, status, txid, scripthash, known_txids in candidates:
history_cache: dict[str, ServerAnswers] = {}
for candidate in candidates:
try:
if scripthash not in history_cache:
history_cache[scripthash] = await client.get_history(scripthash)
found = [entry["tx_hash"] for entry in history_cache[scripthash] if entry.get("tx_hash") in known_txids]
exists = bool(found)
if candidate.scripthash not in history_cache:
history_cache[candidate.scripthash] = await _histories(client, network, candidate.scripthash)
answers = history_cache[candidate.scripthash]
decision = await _decide(network, candidate, answers)
except Exception:
# Transport/server problem — say nothing about this tx and try again on
# the next pass rather than abandoning a tx that may be perfectly alive.
logger.warning("could not check pending_transaction %s (txid %s) against the chain", row_id, txid)
logger.warning(
"could not check pending_transaction %s (txid %s) against the chain",
candidate.row_id,
candidate.txid,
exc_info=True,
)
continue
if decision is None:
continue
async with session_factory() as session:
await begin_immediate(session) # B-88: the re-check and the resolution under one lock
row = await session.get(PendingTransaction, row_id)
if row is None or row.status != status:
continue # something else moved it while we were asking
if exists:
if txid not in found:
row = await session.get(PendingTransaction, candidate.row_id)
# TX-26: the status *and* the current txid must still be what they were when
# the network was asked — a confirmation or an RBF in the meantime changes
# what the answer means.
if row is None or row.status != candidate.status or row.current_txid != candidate.txid:
continue
kind, payload = decision
changed = False
if kind == "resurrected":
await _resurrect(session, row, *payload)
changed = True
elif kind == "present":
if payload and payload != row.current_txid:
# Another version of the row is the one on the network (B-77, TX-19):
# the row follows it, so the inputs and the domain row point at it.
await adopt_version(session, row, found[0])
resolved += 1
await adopt_version(session, row, payload)
changed = True
if row.status == "building":
await _promote(session, row)
resolved += 1
else:
await _abandon(session, row, "not found on chain")
resolved += 1
changed = True
elif kind == "promote":
await _promote(session, row)
changed = True
elif kind == "abandon":
await _abandon(session, row, payload)
changed = True
await session.commit()
resolved += changed
broadcaster.publish()
return resolved
async def _histories(client: ElectrumClient, network, scripthash: str) -> ServerAnswers:
if network is not None:
return await network.get_histories(scripthash)
# No quorum to ask: the active server alone can show a transaction exists, but its
# silence can never be taken for absence (`complete` stays false).
return ServerAnswers(await client.get_history(scripthash), (), 1)
def _entries_for(answers: ServerAnswers, known_txids: set[str]) -> list[tuple[str, int, bool]]:
"""(txid, height, from_active) for every history entry of every answering server that
is one of the row's txids."""
found: list[tuple[str, int, bool]] = []
if answers.active is not None:
found += [(e["tx_hash"], e.get("height") or 0, True) for e in answers.active if e.get("tx_hash") in known_txids]
for history in answers.answered_others:
found += [(e["tx_hash"], e.get("height") or 0, False) for e in history if e.get("tx_hash") in known_txids]
return found
def _confirmed_by_the_quorum(answers: ServerAnswers, known_txids: set[str]) -> tuple[str, int] | None:
"""A txid of the row that a strict majority of the answering *other* servers (at
least the quorum minimum of them) list in a block, with the height they report.
The active server's word is not part of it."""
others = answers.answered_others
if len(others) < max(answers.others_needed, 1):
return None
votes: dict[tuple[str, int], int] = {}
for history in others:
for entry in history:
if entry.get("tx_hash") in known_txids and (entry.get("height") or 0) > 0:
key = (entry["tx_hash"], entry["height"])
votes[key] = votes.get(key, 0) + 1
for key, count in sorted(votes.items(), key=lambda item: -item[1]):
if count * 2 > len(others):
return key
return None
async def _decide(network, candidate: _Candidate, answers: ServerAnswers):
"""What the network says about one row, as ("present", txid | None), ("promote", None),
("abandon", reason), ("resurrected", (txid, height)) or None for "nothing to do or
not enough to tell". Pure reads — nothing is written here."""
if candidate.status == "failed":
confirmed = _confirmed_by_the_quorum(answers, candidate.known_txids)
return ("resurrected", confirmed) if confirmed is not None else None
found = _entries_for(answers, candidate.known_txids)
if found:
# Prefer a version that is mined, then the current one, then any.
mined = [txid for txid, height, _ in found if height > 0]
present = {txid for txid, _, _ in found}
chosen = mined[0] if mined else (candidate.txid if candidate.txid in present else found[0][0])
return ("present", chosen)
if not answers.complete:
logger.warning(
"pending_transaction %s (txid %s): too few servers answered to say the tx is unknown — left as is",
candidate.row_id,
candidate.txid,
)
return None
if network is None:
return None
# No server of the quorum knows any txid of the row. That alone proves nothing (TX-15).
tx = Transaction.parse(bytes.fromhex(candidate.raw_hex))
outpoints = {(vin.txid.hex(), vin.vout) for vin in tx.vin}
spender = await network.confirmed_spender(candidate.scripthash, candidate.known_txids, outpoints)
if spender is not None:
return ("abandon", f"input spent in a block by {spender}")
rebroadcast: ServerAnswers = await network.rebroadcast_everywhere(candidate.raw_hex)
verdicts = rebroadcast.all
if any(v[0] == "accepted" or (v[0] == "refused" and is_already_known(v[1])) for v in verdicts):
# It was alive after all (or is again): the same as having found it.
return ("present", None)
refusals = [v[1].lower() for v in verdicts if v[0] == "refused"]
if rebroadcast.complete and len(refusals) == len(verdicts) and all(
any(marker in message for marker in _MISSING_INPUTS_MARKERS) for message in refusals
):
return ("abandon", f"rebroadcast refused for missing inputs: {refusals[0]}")
if candidate.status == "building" and any(any(marker in m for marker in _FEE_MARKERS) for m in refusals):
return ("promote", None) # not proof: let the RBF bumper accelerate it (TX-17)
return None
def _is_due(row: PendingTransaction, now: datetime) -> bool:
if row.status == "failed":
# B-78 (TX-23, DAT-72): watched every pass for FAILED_WATCH_DAYS from failed_at.
failed_at = row.failed_at or row.updated_at
return failed_at is not None and now < failed_at.replace(tzinfo=timezone.utc) + timedelta(days=FAILED_WATCH_DAYS)
# Deliberately broadcast_at (the *first* broadcast), not last_broadcast_at: an
# RBF bump used to overwrite this same field, which reset this grace period on
# every bump and meant a repeatedly-bumped-but-never-mined tx was never
# abandoned (B-27). tx/broadcast.py:bump_fee now only ever touches
# last_broadcast_at, so this keeps measuring from when the tx first appeared,
# no matter how many times it's since been bumped.
# no matter how many times it's since been bumped. The period only decides from
# when a row is *examined*; giving it up still needs proof (TX-14, TX-17).
grace = _BUILDING_GRACE_SECONDS if row.status == "building" else _ABANDON_AFTER_SECONDS
return now >= row.broadcast_at.replace(tzinfo=timezone.utc) + timedelta(seconds=grace)
async def _resurrect(session: AsyncSession, row: PendingTransaction, txid: str, height: int) -> None:
"""A row given up for dead was mined (B-78, TX-23, DAT-72). The row becomes
`confirmed` and the lottery is paused for the operator: the domain rows the
abandonment already rolled back (a deleted participant, a failed withdrawal, released
coins, a rebuilt payout) are *not* put back by the system, which cannot know what has
happened to them since."""
if txid != row.current_txid:
version_raw = None
version = await session.scalar(
select(PendingTransactionTxid).where(
PendingTransactionTxid.txid == txid, PendingTransactionTxid.pending_transaction_id == row.id
)
)
if version is not None and version.raw_hex:
version_raw = version.raw_hex
row.current_txid = txid
if version_raw:
row.raw_tx_hex = version_raw
row.status = "confirmed"
await write_audit_log(
session,
"pending_tx_resurrected",
{"pending_transaction_id": row.id, "kind": row.kind, "txid": txid, "block_height": height},
user_id=row.user_id,
round_id=row.round_id,
)
await pause_automatically(session, "resurrected_tx")
logger.error(
"%s pending_transaction %s was given up as failed but tx %s is in block %s — lottery paused for the operator",
row.kind,
row.id,
txid,
height,
)
async def _promote(session: AsyncSession, row: PendingTransaction) -> None:
"""The tx did make it onto the chain before we died — finish what phase 2 of
place_bet/request_withdrawal would have done."""
@@ -190,8 +384,10 @@ async def _promote(session: AsyncSession, row: PendingTransaction) -> None:
async def _abandon(session: AsyncSession, row: PendingTransaction, reason: str) -> None:
"""The tx is gone for good. Release whatever it reserved so the funds come back,
and roll the domain row back to something truthful."""
reason = reason[:128]
row.status = "failed"
row.failure_reason = reason[:128]
row.failure_reason = reason
row.failed_at = datetime.now(timezone.utc) # B-78: starts the 14-day watch
released = await _release_inputs(session, row, await txids_of(session, row))
+2
View File
@@ -1,4 +1,5 @@
import logging
from datetime import datetime, timezone
from embit import script
from sqlalchemy import select
@@ -246,6 +247,7 @@ async def _release_failed_withdrawal(
withdrawal.txid = None
pending.status = "failed"
pending.failure_reason = reason[:128]
pending.failed_at = datetime.now(timezone.utc) # B-78 (DAT-36): starts the 14-day watch
await recompute_balance(session, user_id)
await write_audit_log(
session,