Abandon a transaction only on proof, and keep watching failed rows (B-78, TX-15, TX-17, TX-23, TX-26, ELC-28, DAT-33, DAT-36, DAT-69, DAT-72, AUD-11, AUD-12, AUD-31, AUD-39, API-32)
reconcile_once abandoned a row when the *active* server's history lacked its txid:
after 120s for `building` (which a restart onto a different server makes likely),
after 6h for `pending`, and all at once after a forward clock jump. It re-checked the
status but not the txid, so it raced the bumper, and a `failed` row was never looked at
again.
- The reconciler asks every server (ElectrumListener.get_histories, active + others) for
the history of the row's address and looks for every txid of the row. Present on any
server -> promoted (adopting the version found). Absent is only "no server of the
quorum knows it" and needs the active server plus the quorum minimum of the others to
have answered; even then nothing is released without proof:
(a) an input spent in a block by a transaction that is none of the row's versions,
agreed by a strict majority of a quorum of the other servers (confirmed_spender), or
(b) a rebroadcast of the latest raw tx to every server (rebroadcast_everywhere)
refused by all of them, explicitly, for missing/spent inputs.
Accepted or "already known" by any server means alive; a fee refusal is not proof
(`building` -> `pending` for the bumper, `pending` untouched); too few answers or
anything else leaves the row alone. failure_reason names the proof.
- The write re-reads status *and* current_txid under begin_immediate (TX-26).
- pending_transactions.failed_at (set by _abandon and _release_failed_withdrawal) starts
a 14-day watch: a failed row that a majority of the other servers (at least the quorum
minimum) list in a block becomes `confirmed`, writes pending_tx_resurrected once and
pauses the lottery with paused_reason `resurrected_tx`; linked rows are not corrected.
The confirmation is by quorum of histories for now; merkle proofs come with B-83.
- round_config.paused_reason (migration b7d2e41c9a05, with the failed_at one): `operator`
from POST /admin/pause (which accepts an optional `note` and never overwrites an
automatic reason), cleared by resume, which logs the reason it lifted; exposed on
/admin/config and GET /rounds/current. rounds/pause.py:pause_automatically is the
system's way in.
- PendingTransactionReconciler takes the listener as `network`; with none it can promote
but never abandon.
Tests changed because the spec changed: the three reconciler tests that expect an
abandonment (and the one in test_tx_versions) now supply a network that proves the
transaction is gone.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f936d6660c
commit
a4b160e240
29 files changed
+1280
-201
No files matched your search
+26
-4
@@ -13,6 +13,7 @@ from app.config import settings
|
||||
from app.db.models import AuditLog, BugReport, PendingTransaction, Round, User
|
||||
from app.db.session import get_session
|
||||
from app.rounds.config import get_round_config
|
||||
from app.rounds.pause import AUTOMATIC_REASONS, OPERATOR_REASON
|
||||
from app.wallet.address import is_valid_plm_address
|
||||
from app.wallet.hd import derive_user_wif
|
||||
from app.wallet.psbt_builder import MAX_FEE_RATE_SAT_VB
|
||||
@@ -56,6 +57,7 @@ class RoundConfigResponse(BaseModel):
|
||||
rbf_timeout_seconds: int
|
||||
draw_animation_seconds: int
|
||||
paused: bool
|
||||
paused_reason: str | None = None # B-78 (DAT-69): why, or None when not paused
|
||||
|
||||
|
||||
class RoundConfigUpdate(BaseModel):
|
||||
@@ -93,6 +95,7 @@ class RoundConfigUpdate(BaseModel):
|
||||
def _config_response(config) -> RoundConfigResponse:
|
||||
fields = {field: getattr(config, field) for field in _CONFIG_FIELDS}
|
||||
fields["paused"] = config.paused
|
||||
fields["paused_reason"] = config.paused_reason
|
||||
return RoundConfigResponse(**fields)
|
||||
|
||||
|
||||
@@ -128,14 +131,29 @@ async def update_config(
|
||||
return _config_response(config)
|
||||
|
||||
|
||||
class PauseRequest(BaseModel):
|
||||
note: str | None = Field(default=None, max_length=256)
|
||||
|
||||
|
||||
@router.post("/pause", response_model=RoundConfigResponse, dependencies=[Depends(require_admin)])
|
||||
async def pause_lottery(session: AsyncSession = Depends(get_session)) -> RoundConfigResponse:
|
||||
async def pause_lottery(
|
||||
body: PauseRequest | None = None, session: AsyncSession = Depends(get_session)
|
||||
) -> RoundConfigResponse:
|
||||
"""Maintenance switch: the round in progress (if any) still closes, draws,
|
||||
and pays out its winner normally — only opening the *next* round is
|
||||
suppressed until /admin/resume is called (rounds/service.py)."""
|
||||
suppressed until /admin/resume is called (rounds/service.py).
|
||||
|
||||
B-78 (DAT-69, API-32): the reason is "operator", except that an automatic reason
|
||||
already in force stays — it is the one that needs a look. The audit row always
|
||||
says `operator` for this call."""
|
||||
config = await get_round_config(session)
|
||||
if not (config.paused and config.paused_reason in AUTOMATIC_REASONS):
|
||||
config.paused_reason = OPERATOR_REASON
|
||||
config.paused = True
|
||||
await write_audit_log(session, "lottery_paused", {})
|
||||
payload = {"reason": OPERATOR_REASON}
|
||||
if body is not None and body.note:
|
||||
payload["note"] = body.note
|
||||
await write_audit_log(session, "lottery_paused", payload)
|
||||
await session.commit()
|
||||
return _config_response(config)
|
||||
|
||||
@@ -143,8 +161,12 @@ async def pause_lottery(session: AsyncSession = Depends(get_session)) -> RoundCo
|
||||
@router.post("/resume", response_model=RoundConfigResponse, dependencies=[Depends(require_admin)])
|
||||
async def resume_lottery(session: AsyncSession = Depends(get_session)) -> RoundConfigResponse:
|
||||
config = await get_round_config(session)
|
||||
# B-78 (AUD-12): the reason the resume took away, so the log says what the operator
|
||||
# agreed to resume (None when the lottery was not paused).
|
||||
lifted = config.paused_reason if config.paused else None
|
||||
config.paused = False
|
||||
await write_audit_log(session, "lottery_resumed", {})
|
||||
config.paused_reason = None
|
||||
await write_audit_log(session, "lottery_resumed", {"reason": lifted})
|
||||
await session.commit()
|
||||
return _config_response(config)
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ class CurrentRoundResponse(BaseModel):
|
||||
draw_waiting_since: str | None = None
|
||||
chain_tip_height: int | None = None
|
||||
lottery_paused: bool = False
|
||||
paused_reason: str | None = None # B-78 (DAT-69): operator / resurrected_tx / ..., None if not paused
|
||||
# B-66: false while the instance is missing configuration a round cannot run
|
||||
# without (today: fee_address) — no round will open until it's set, so this is
|
||||
# the difference between "wait, the next round is coming" and "nothing is coming
|
||||
@@ -147,6 +148,7 @@ async def current_round(
|
||||
draw_animation_seconds=config.draw_animation_seconds,
|
||||
chain_tip_height=chain_tip_height,
|
||||
lottery_paused=config.paused,
|
||||
paused_reason=config.paused_reason,
|
||||
lottery_configured=rounds_can_open(config),
|
||||
)
|
||||
|
||||
@@ -225,6 +227,7 @@ async def current_round(
|
||||
draw_waiting_since=isoformat_utc(round_.drawing_started_at) if round_.status == "drawing" else None,
|
||||
chain_tip_height=chain_tip_height,
|
||||
lottery_paused=config.paused,
|
||||
paused_reason=config.paused_reason,
|
||||
lottery_configured=rounds_can_open(config),
|
||||
user_played=user_played,
|
||||
)
|
||||
@@ -155,6 +155,12 @@ class RoundConfig(Base):
|
||||
# completion (closes, draws, pays out the winner) but no new round is opened
|
||||
# afterwards — see rounds/service.py:open_new_round_if_needed.
|
||||
paused: Mapped[bool] = mapped_column(default=False)
|
||||
# B-78 (DAT-69): why the lottery is paused — NULL exactly when `paused` is false.
|
||||
# "operator" (POST /admin/pause) or an automatic reason the system wrote itself
|
||||
# ("resurrected_tx" so far; "reorg" and "payout_blocked" belong to B-85 and B-82).
|
||||
# An automatic reason is never overwritten by an operator pause, so the cause that
|
||||
# needs a look is not lost; POST /admin/resume clears whichever it is.
|
||||
paused_reason: Mapped[str | None] = mapped_column(String(16), default=None)
|
||||
updated_at: Mapped[datetime] = mapped_column(default=utcnow, onupdate=utcnow)
|
||||
|
||||
|
||||
@@ -197,6 +203,9 @@ class PendingTransaction(Base):
|
||||
attempt_count: Mapped[int] = mapped_column(default=1)
|
||||
# Why the reconciler gave up on this tx — operator-facing, only set on "failed".
|
||||
failure_reason: Mapped[str | None] = mapped_column(String(128), default=None)
|
||||
# B-78 (DAT-72): when the row went to "failed". A failed row stays watched for
|
||||
# FAILED_WATCH_DAYS from here: a transaction given up for dead can still confirm.
|
||||
failed_at: Mapped[datetime | None] = mapped_column(default=None)
|
||||
updated_at: Mapped[datetime] = mapped_column(default=utcnow, onupdate=utcnow)
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
|
||||
from embit.transaction import Transaction
|
||||
@@ -16,6 +17,7 @@ from app.deposits.service import (
|
||||
)
|
||||
from app.config import MIN_QUORUM_RESPONSES
|
||||
from app.electrum.client import ElectrumClient, ElectrumEndpoint, ElectrumRpcError
|
||||
from app.electrum.quorum import ServerAnswers
|
||||
from app.electrum.scripthash import address_to_scripthash
|
||||
from app.rounds.draw import (
|
||||
HeaderValidationError,
|
||||
@@ -497,6 +499,78 @@ class ElectrumListener:
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._background_tasks.discard)
|
||||
|
||||
# --- What the network knows of a transaction (B-78, TX-15, TX-17) -------------------
|
||||
|
||||
async def _ask_everyone(self, ask: Callable[[ElectrumClient], "asyncio.Future"]) -> ServerAnswers:
|
||||
active, others = await asyncio.gather(self._ask_active_server(ask), self._ask_other_servers(ask))
|
||||
return ServerAnswers(active, tuple(others), self._minimum_other_answers())
|
||||
|
||||
async def get_histories(self, scripthash: str) -> ServerAnswers:
|
||||
"""`blockchain.scripthash.get_history` of `scripthash` from the active server and
|
||||
every other one: each answer is the list of {"tx_hash", "height"} entries."""
|
||||
|
||||
async def _ask(client: ElectrumClient) -> list | None:
|
||||
try:
|
||||
history = await client.get_history(scripthash)
|
||||
except Exception:
|
||||
return None
|
||||
return history if isinstance(history, list) else None
|
||||
|
||||
return await self._ask_everyone(_ask)
|
||||
|
||||
async def rebroadcast_everywhere(self, raw_tx_hex: str) -> ServerAnswers:
|
||||
"""Send `raw_tx_hex` to the active server and every other one and report what each
|
||||
said: ("accepted", "") or ("refused", <its message>) — an explicit JSON-RPC error —
|
||||
and None for a server that gave no answer at all (TX-17). A tx the servers already
|
||||
have comes back as a refusal with an "already known" message; the caller reads it."""
|
||||
|
||||
async def _ask(client: ElectrumClient) -> tuple | None:
|
||||
try:
|
||||
await client.broadcast(raw_tx_hex)
|
||||
except ElectrumRpcError as exc:
|
||||
return ("refused", exc.message)
|
||||
except Exception:
|
||||
return None
|
||||
return ("accepted", "")
|
||||
|
||||
return await self._ask_everyone(_ask)
|
||||
|
||||
async def confirmed_spender(
|
||||
self, scripthash: str, own_txids: set[str], outpoints: set[tuple[str, int]]
|
||||
) -> str | None:
|
||||
"""The txid of a transaction, none of the row's own versions, that spends one of
|
||||
`outpoints` and is **in a block** — the first proof by which a row may be given
|
||||
up (TX-17): its inputs went elsewhere. Asked of the *other* servers only (the
|
||||
active one is what a hostile peer would control) and returned only when at least
|
||||
the quorum minimum answered and a strict majority of the answerers name the same
|
||||
txid. Each server looks through the confirmed history of the address the row
|
||||
spends from.
|
||||
"""
|
||||
|
||||
async def _ask(client: ElectrumClient) -> str | None:
|
||||
history = await client.get_history(scripthash)
|
||||
for entry in history:
|
||||
txid = entry.get("tx_hash")
|
||||
if txid in own_txids or (entry.get("height") or 0) <= 0:
|
||||
continue
|
||||
raw = await client.get_transaction(txid, False)
|
||||
tx = Transaction.parse(bytes.fromhex(raw))
|
||||
if tx.txid().hex() != txid:
|
||||
continue
|
||||
if any((vin.txid.hex(), vin.vout) in outpoints for vin in tx.vin):
|
||||
return txid
|
||||
return "" # looked everywhere: no conflicting transaction
|
||||
|
||||
answers = await self._ask_other_servers(_ask)
|
||||
answered = [answer for answer in answers if answer is not None]
|
||||
if len(answered) < max(self._minimum_other_answers(), 1):
|
||||
return None
|
||||
counts = Counter(answer for answer in answered if answer)
|
||||
if not counts:
|
||||
return None
|
||||
txid, votes = counts.most_common(1)[0]
|
||||
return txid if votes * 2 > len(answered) else None
|
||||
|
||||
async def corroborate_header(self, height: int, expected_hash: str) -> bool:
|
||||
"""B-28: is `expected_hash` — the header our own active connection
|
||||
reported for `height` — corroborated by other configured servers before
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""What a question put to every server came back with (B-78).
|
||||
|
||||
`active` is the answer of the connection the platform is running on, `others` those of
|
||||
each other configured server in configuration order; `None` stands for "no usable
|
||||
answer" (unreachable, timed out, an error that is not an answer)."""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
|
||||
class ServerAnswers(NamedTuple):
|
||||
active: object | None
|
||||
others: tuple
|
||||
others_needed: int # how many of `others` must have answered for the network to count
|
||||
|
||||
@property
|
||||
def answered_others(self) -> list:
|
||||
return [answer for answer in self.others if answer is not None]
|
||||
|
||||
@property
|
||||
def complete(self) -> bool:
|
||||
"""Enough of the network answered for its silence to mean something: the active
|
||||
server and at least `others_needed` of the others."""
|
||||
return self.active is not None and len(self.answered_others) >= self.others_needed
|
||||
|
||||
@property
|
||||
def all(self) -> list:
|
||||
return ([] if self.active is None else [self.active]) + self.answered_others
|
||||
+1
-1
@@ -66,7 +66,7 @@ async def lifespan(app: FastAPI):
|
||||
bumper = RbfBumper(AsyncSessionLocal, lambda: listener.client, network=listener)
|
||||
# Resolves in-flight transactions against the chain — the piece that lets the
|
||||
# system recover on its own from a broadcast that never confirmed (B-04/B-08).
|
||||
reconciler = PendingTransactionReconciler(AsyncSessionLocal, lambda: listener.client)
|
||||
reconciler = PendingTransactionReconciler(AsyncSessionLocal, lambda: listener.client, network=listener)
|
||||
# Periodic safety net for deposit crediting/external-spend detection,
|
||||
# independent of scripthash-change notifications — catches a subscription
|
||||
# silently lost on an otherwise healthy connection (B-30).
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Pausing the lottery from code (B-78, DAT-69, AUD-11).
|
||||
|
||||
A pause only stops the *next* round from opening (rounds/service.py:rounds_can_open); a
|
||||
round in progress still closes, draws and pays. Besides the operator's own switch there
|
||||
are pauses the system raises itself when something needs a human look — a transaction
|
||||
given up for dead that confirmed after all (`resurrected_tx`, B-78) and, in later
|
||||
findings, a deep reorg (`reorg`, B-85) and a payout whose winning bet left the chain
|
||||
(`payout_blocked`, B-82).
|
||||
"""
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.audit.log import write_audit_log
|
||||
from app.rounds.config import get_round_config
|
||||
|
||||
OPERATOR_REASON = "operator"
|
||||
AUTOMATIC_REASONS = ("reorg", "payout_blocked", "resurrected_tx")
|
||||
|
||||
|
||||
async def pause_automatically(session: AsyncSession, reason: str) -> None:
|
||||
"""Pause the lottery for an automatic `reason` in the caller's transaction and write
|
||||
the `lottery_paused` row. An automatic reason is written even over a pause that is
|
||||
already active (DAT-69): the cause that needs checking must not be hidden behind an
|
||||
earlier one, and an operator pause is never what overrides it."""
|
||||
if reason not in AUTOMATIC_REASONS:
|
||||
raise ValueError(f"not an automatic pause reason: {reason!r}")
|
||||
config = await get_round_config(session)
|
||||
config.paused = True
|
||||
config.paused_reason = reason
|
||||
await write_audit_log(session, "lottery_paused", {"reason": reason})
|
||||
+244
-48
@@ -13,15 +13,26 @@ balance simply lost them, with no path back short of editing the DB by hand
|
||||
What it does, per PendingTransaction that isn't already terminal:
|
||||
|
||||
* status "building" — we crashed (or were killed) between writing the row and
|
||||
broadcasting. Ask the chain: if the tx is there after all, promote everything
|
||||
to its live state; if it isn't, release the UTXOs and undo the intent.
|
||||
broadcasting. Ask the network: if any server knows one of the row's txids,
|
||||
promote everything to its live state.
|
||||
* status "pending" — broadcast, still unconfirmed. Left alone until it has been
|
||||
unconfirmed for `_ABANDON_AFTER_SECONDS`, since absence from one server's
|
||||
mempool is not proof of death; only then is it abandoned like the above.
|
||||
unconfirmed for `_ABANDON_AFTER_SECONDS`, and even then only examined.
|
||||
|
||||
Deliberately conservative: it never touches a tx the chain knows about, and the
|
||||
grace period is long (multiples of the RBF timeout) so a slow-but-alive tx is
|
||||
bumped by RbfBumper rather than abandoned here.
|
||||
and then *nothing is given up without proof* (B-78, ADR-51, TX-17). A row is
|
||||
abandoned — its coins released — only when
|
||||
|
||||
(a) one of its inputs is spent **in a block** by a transaction that is none of its
|
||||
own versions, and the other servers agree; or
|
||||
(b) no server of the quorum knows any txid of the row **and** sending its latest
|
||||
raw transaction to every server is refused, explicitly and by all of them, for
|
||||
missing or already-spent inputs.
|
||||
|
||||
Absence from one server's history proves nothing — after a restart onto a different
|
||||
server that is the normal state of a tx the new server has not seen — and neither
|
||||
does the age of the row. A refusal for fee reasons is no proof either: such a row is
|
||||
promoted (building) or left to the RBF bumper (pending). A transport failure never
|
||||
resolves anything. Failed rows stay watched for `FAILED_WATCH_DAYS`: if one of them
|
||||
confirms after all, the lottery is paused for the operator (TX-23).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -35,11 +46,21 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.audit.log import write_audit_log
|
||||
from app.db.base import begin_immediate
|
||||
from app.db.models import PendingTransaction, Round, RoundParticipant, UtxoEvent, Withdrawal
|
||||
from app.db.models import (
|
||||
PendingTransaction,
|
||||
PendingTransactionTxid,
|
||||
Round,
|
||||
RoundParticipant,
|
||||
UtxoEvent,
|
||||
Withdrawal,
|
||||
)
|
||||
from app.electrum.client import ElectrumClient
|
||||
from app.electrum.quorum import ServerAnswers
|
||||
from app.electrum.scripthash import address_to_scripthash
|
||||
from app.rounds.events import broadcaster
|
||||
from app.rounds.pause import pause_automatically
|
||||
from app.tx.broadcast import adopt_version
|
||||
from app.tx.outcome import is_already_known
|
||||
from app.tx.pending_address import own_address_for
|
||||
from app.tx.versions import txids_by_row, txids_of
|
||||
from app.wallet.balance import recompute_balance
|
||||
@@ -53,14 +74,29 @@ _POLL_INTERVAL_SECONDS = 120
|
||||
_BUILDING_GRACE_SECONDS = 120
|
||||
|
||||
# A "pending" row was accepted by a node once. Give it a wide margin — the RBF
|
||||
# bumper gets several attempts inside this window — before concluding it's gone.
|
||||
# bumper gets several attempts inside this window — before it is even examined.
|
||||
_ABANDON_AFTER_SECONDS = 6 * 60 * 60
|
||||
|
||||
# How long a failed row keeps being looked at (B-78, DAT-72): a transaction given up
|
||||
# for dead can still arrive in a block.
|
||||
FAILED_WATCH_DAYS = 14
|
||||
|
||||
# What a node says when the transaction's inputs do not exist or are spent already.
|
||||
_MISSING_INPUTS_MARKERS = ("missing-inputs", "missingorspent", "missing inputs", "inputs-missing")
|
||||
# What it says when only the fee is the problem: not proof of anything.
|
||||
_FEE_MARKERS = ("min relay fee", "mempool min fee", "insufficient fee", "fee not met", "fee too low", "fee rate")
|
||||
|
||||
|
||||
class PendingTransactionReconciler:
|
||||
def __init__(self, session_factory: async_sessionmaker, get_client: Callable[[], ElectrumClient | None]):
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker,
|
||||
get_client: Callable[[], ElectrumClient | None],
|
||||
network=None,
|
||||
):
|
||||
self._session_factory = session_factory
|
||||
self._get_client = get_client
|
||||
self._network = network # the ElectrumListener: histories of every server, rebroadcast, spenders
|
||||
|
||||
async def run(self) -> None:
|
||||
# Runs once promptly at startup: a crash mid-broadcast is exactly the case
|
||||
@@ -69,7 +105,7 @@ class PendingTransactionReconciler:
|
||||
client = self._get_client()
|
||||
if client is not None:
|
||||
try:
|
||||
await reconcile_once(self._session_factory, client)
|
||||
await reconcile_once(self._session_factory, client, self._network)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
@@ -77,31 +113,41 @@ class PendingTransactionReconciler:
|
||||
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def reconcile_once(session_factory: async_sessionmaker, client: ElectrumClient) -> int:
|
||||
"""Returns how many rows were resolved (promoted or abandoned).
|
||||
class _Candidate:
|
||||
__slots__ = ("row_id", "status", "txid", "scripthash", "known_txids", "raw_hex")
|
||||
|
||||
Existence is decided by checking whether a row's own address's history
|
||||
(blockchain.scripthash.get_history) includes its txid at all — mempool or
|
||||
mined — rather than asking blockchain.transaction.get for a verbose reply
|
||||
(B-41): several Electrum server implementations and versions reject the
|
||||
verbose flag outright, and the previous substring-matching on the error
|
||||
text (looking for "missing", "not found", ...) was fragile as the basis for
|
||||
a decision that releases funds. A transport failure fetching history still
|
||||
raises and leaves the row alone until next time — get_history not
|
||||
returning our txid is the only thing that means "gone". History is cached
|
||||
per scripthash within one pass, since the payout rows of one round share its
|
||||
pool address (B-80).
|
||||
def __init__(self, row_id, status, txid, scripthash, known_txids, raw_hex):
|
||||
self.row_id = row_id
|
||||
self.status = status
|
||||
self.txid = txid
|
||||
self.scripthash = scripthash
|
||||
self.known_txids = known_txids
|
||||
self.raw_hex = raw_hex
|
||||
|
||||
|
||||
async def reconcile_once(
|
||||
session_factory: async_sessionmaker, client: ElectrumClient, network=None
|
||||
) -> int:
|
||||
"""Returns how many rows were resolved (promoted, abandoned, realigned or
|
||||
resurrected).
|
||||
|
||||
Existence is decided by asking every server of the quorum for the history of the
|
||||
row's own address (blockchain.scripthash.get_history) and looking for **every** txid
|
||||
the row has had, mempool or mined (B-41, B-77, B-78) — never for a verbose
|
||||
blockchain.transaction.get reply, which several servers reject outright, and never
|
||||
on error text. History is cached per scripthash within one pass, since the payout
|
||||
rows of one round share its pool address (B-80). With no `network` only the active
|
||||
server is asked, which can promote a row but never abandon one.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with session_factory() as session:
|
||||
rows = (
|
||||
await session.scalars(
|
||||
select(PendingTransaction).where(PendingTransaction.status.in_(("building", "pending")))
|
||||
select(PendingTransaction).where(PendingTransaction.status.in_(("building", "pending", "failed")))
|
||||
)
|
||||
).all()
|
||||
candidates = []
|
||||
# B-77 (TX-15, TX-19): a row's every txid, not only the current one.
|
||||
txids_by_id = await txids_by_row(session, {row.id: row.current_txid for row in rows})
|
||||
candidates: list[_Candidate] = []
|
||||
for row in rows:
|
||||
if not _is_due(row, now):
|
||||
continue
|
||||
@@ -111,56 +157,204 @@ async def reconcile_once(session_factory: async_sessionmaker, client: ElectrumCl
|
||||
except Exception:
|
||||
logger.exception("could not derive the address for pending_transaction %s", row.id)
|
||||
continue
|
||||
candidates.append((row.id, row.status, row.current_txid, scripthash, txids_by_id[row.id]))
|
||||
candidates.append(
|
||||
_Candidate(row.id, row.status, row.current_txid, scripthash, txids_by_id[row.id], row.raw_tx_hex)
|
||||
)
|
||||
|
||||
resolved = 0
|
||||
history_cache: dict[str, list[dict]] = {}
|
||||
for row_id, status, txid, scripthash, known_txids in candidates:
|
||||
history_cache: dict[str, ServerAnswers] = {}
|
||||
for candidate in candidates:
|
||||
try:
|
||||
if scripthash not in history_cache:
|
||||
history_cache[scripthash] = await client.get_history(scripthash)
|
||||
found = [entry["tx_hash"] for entry in history_cache[scripthash] if entry.get("tx_hash") in known_txids]
|
||||
exists = bool(found)
|
||||
if candidate.scripthash not in history_cache:
|
||||
history_cache[candidate.scripthash] = await _histories(client, network, candidate.scripthash)
|
||||
answers = history_cache[candidate.scripthash]
|
||||
decision = await _decide(network, candidate, answers)
|
||||
except Exception:
|
||||
# Transport/server problem — say nothing about this tx and try again on
|
||||
# the next pass rather than abandoning a tx that may be perfectly alive.
|
||||
logger.warning("could not check pending_transaction %s (txid %s) against the chain", row_id, txid)
|
||||
logger.warning(
|
||||
"could not check pending_transaction %s (txid %s) against the chain",
|
||||
candidate.row_id,
|
||||
candidate.txid,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
if decision is None:
|
||||
continue
|
||||
|
||||
async with session_factory() as session:
|
||||
await begin_immediate(session) # B-88: the re-check and the resolution under one lock
|
||||
row = await session.get(PendingTransaction, row_id)
|
||||
if row is None or row.status != status:
|
||||
continue # something else moved it while we were asking
|
||||
if exists:
|
||||
if txid not in found:
|
||||
row = await session.get(PendingTransaction, candidate.row_id)
|
||||
# TX-26: the status *and* the current txid must still be what they were when
|
||||
# the network was asked — a confirmation or an RBF in the meantime changes
|
||||
# what the answer means.
|
||||
if row is None or row.status != candidate.status or row.current_txid != candidate.txid:
|
||||
continue
|
||||
kind, payload = decision
|
||||
changed = False
|
||||
if kind == "resurrected":
|
||||
await _resurrect(session, row, *payload)
|
||||
changed = True
|
||||
elif kind == "present":
|
||||
if payload and payload != row.current_txid:
|
||||
# Another version of the row is the one on the network (B-77, TX-19):
|
||||
# the row follows it, so the inputs and the domain row point at it.
|
||||
await adopt_version(session, row, found[0])
|
||||
resolved += 1
|
||||
await adopt_version(session, row, payload)
|
||||
changed = True
|
||||
if row.status == "building":
|
||||
await _promote(session, row)
|
||||
resolved += 1
|
||||
else:
|
||||
await _abandon(session, row, "not found on chain")
|
||||
resolved += 1
|
||||
changed = True
|
||||
elif kind == "promote":
|
||||
await _promote(session, row)
|
||||
changed = True
|
||||
elif kind == "abandon":
|
||||
await _abandon(session, row, payload)
|
||||
changed = True
|
||||
await session.commit()
|
||||
resolved += changed
|
||||
broadcaster.publish()
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
async def _histories(client: ElectrumClient, network, scripthash: str) -> ServerAnswers:
|
||||
if network is not None:
|
||||
return await network.get_histories(scripthash)
|
||||
# No quorum to ask: the active server alone can show a transaction exists, but its
|
||||
# silence can never be taken for absence (`complete` stays false).
|
||||
return ServerAnswers(await client.get_history(scripthash), (), 1)
|
||||
|
||||
|
||||
def _entries_for(answers: ServerAnswers, known_txids: set[str]) -> list[tuple[str, int, bool]]:
|
||||
"""(txid, height, from_active) for every history entry of every answering server that
|
||||
is one of the row's txids."""
|
||||
found: list[tuple[str, int, bool]] = []
|
||||
if answers.active is not None:
|
||||
found += [(e["tx_hash"], e.get("height") or 0, True) for e in answers.active if e.get("tx_hash") in known_txids]
|
||||
for history in answers.answered_others:
|
||||
found += [(e["tx_hash"], e.get("height") or 0, False) for e in history if e.get("tx_hash") in known_txids]
|
||||
return found
|
||||
|
||||
|
||||
def _confirmed_by_the_quorum(answers: ServerAnswers, known_txids: set[str]) -> tuple[str, int] | None:
|
||||
"""A txid of the row that a strict majority of the answering *other* servers (at
|
||||
least the quorum minimum of them) list in a block, with the height they report.
|
||||
The active server's word is not part of it."""
|
||||
others = answers.answered_others
|
||||
if len(others) < max(answers.others_needed, 1):
|
||||
return None
|
||||
votes: dict[tuple[str, int], int] = {}
|
||||
for history in others:
|
||||
for entry in history:
|
||||
if entry.get("tx_hash") in known_txids and (entry.get("height") or 0) > 0:
|
||||
key = (entry["tx_hash"], entry["height"])
|
||||
votes[key] = votes.get(key, 0) + 1
|
||||
for key, count in sorted(votes.items(), key=lambda item: -item[1]):
|
||||
if count * 2 > len(others):
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
async def _decide(network, candidate: _Candidate, answers: ServerAnswers):
|
||||
"""What the network says about one row, as ("present", txid | None), ("promote", None),
|
||||
("abandon", reason), ("resurrected", (txid, height)) or None for "nothing to do or
|
||||
not enough to tell". Pure reads — nothing is written here."""
|
||||
if candidate.status == "failed":
|
||||
confirmed = _confirmed_by_the_quorum(answers, candidate.known_txids)
|
||||
return ("resurrected", confirmed) if confirmed is not None else None
|
||||
|
||||
found = _entries_for(answers, candidate.known_txids)
|
||||
if found:
|
||||
# Prefer a version that is mined, then the current one, then any.
|
||||
mined = [txid for txid, height, _ in found if height > 0]
|
||||
present = {txid for txid, _, _ in found}
|
||||
chosen = mined[0] if mined else (candidate.txid if candidate.txid in present else found[0][0])
|
||||
return ("present", chosen)
|
||||
|
||||
if not answers.complete:
|
||||
logger.warning(
|
||||
"pending_transaction %s (txid %s): too few servers answered to say the tx is unknown — left as is",
|
||||
candidate.row_id,
|
||||
candidate.txid,
|
||||
)
|
||||
return None
|
||||
if network is None:
|
||||
return None
|
||||
|
||||
# No server of the quorum knows any txid of the row. That alone proves nothing (TX-15).
|
||||
tx = Transaction.parse(bytes.fromhex(candidate.raw_hex))
|
||||
outpoints = {(vin.txid.hex(), vin.vout) for vin in tx.vin}
|
||||
spender = await network.confirmed_spender(candidate.scripthash, candidate.known_txids, outpoints)
|
||||
if spender is not None:
|
||||
return ("abandon", f"input spent in a block by {spender}")
|
||||
|
||||
rebroadcast: ServerAnswers = await network.rebroadcast_everywhere(candidate.raw_hex)
|
||||
verdicts = rebroadcast.all
|
||||
if any(v[0] == "accepted" or (v[0] == "refused" and is_already_known(v[1])) for v in verdicts):
|
||||
# It was alive after all (or is again): the same as having found it.
|
||||
return ("present", None)
|
||||
refusals = [v[1].lower() for v in verdicts if v[0] == "refused"]
|
||||
if rebroadcast.complete and len(refusals) == len(verdicts) and all(
|
||||
any(marker in message for marker in _MISSING_INPUTS_MARKERS) for message in refusals
|
||||
):
|
||||
return ("abandon", f"rebroadcast refused for missing inputs: {refusals[0]}")
|
||||
if candidate.status == "building" and any(any(marker in m for marker in _FEE_MARKERS) for m in refusals):
|
||||
return ("promote", None) # not proof: let the RBF bumper accelerate it (TX-17)
|
||||
return None
|
||||
|
||||
|
||||
def _is_due(row: PendingTransaction, now: datetime) -> bool:
|
||||
if row.status == "failed":
|
||||
# B-78 (TX-23, DAT-72): watched every pass for FAILED_WATCH_DAYS from failed_at.
|
||||
failed_at = row.failed_at or row.updated_at
|
||||
return failed_at is not None and now < failed_at.replace(tzinfo=timezone.utc) + timedelta(days=FAILED_WATCH_DAYS)
|
||||
# Deliberately broadcast_at (the *first* broadcast), not last_broadcast_at: an
|
||||
# RBF bump used to overwrite this same field, which reset this grace period on
|
||||
# every bump and meant a repeatedly-bumped-but-never-mined tx was never
|
||||
# abandoned (B-27). tx/broadcast.py:bump_fee now only ever touches
|
||||
# last_broadcast_at, so this keeps measuring from when the tx first appeared,
|
||||
# no matter how many times it's since been bumped.
|
||||
# no matter how many times it's since been bumped. The period only decides from
|
||||
# when a row is *examined*; giving it up still needs proof (TX-14, TX-17).
|
||||
grace = _BUILDING_GRACE_SECONDS if row.status == "building" else _ABANDON_AFTER_SECONDS
|
||||
return now >= row.broadcast_at.replace(tzinfo=timezone.utc) + timedelta(seconds=grace)
|
||||
|
||||
|
||||
async def _resurrect(session: AsyncSession, row: PendingTransaction, txid: str, height: int) -> None:
|
||||
"""A row given up for dead was mined (B-78, TX-23, DAT-72). The row becomes
|
||||
`confirmed` and the lottery is paused for the operator: the domain rows the
|
||||
abandonment already rolled back (a deleted participant, a failed withdrawal, released
|
||||
coins, a rebuilt payout) are *not* put back by the system, which cannot know what has
|
||||
happened to them since."""
|
||||
if txid != row.current_txid:
|
||||
version_raw = None
|
||||
version = await session.scalar(
|
||||
select(PendingTransactionTxid).where(
|
||||
PendingTransactionTxid.txid == txid, PendingTransactionTxid.pending_transaction_id == row.id
|
||||
)
|
||||
)
|
||||
if version is not None and version.raw_hex:
|
||||
version_raw = version.raw_hex
|
||||
row.current_txid = txid
|
||||
if version_raw:
|
||||
row.raw_tx_hex = version_raw
|
||||
row.status = "confirmed"
|
||||
await write_audit_log(
|
||||
session,
|
||||
"pending_tx_resurrected",
|
||||
{"pending_transaction_id": row.id, "kind": row.kind, "txid": txid, "block_height": height},
|
||||
user_id=row.user_id,
|
||||
round_id=row.round_id,
|
||||
)
|
||||
await pause_automatically(session, "resurrected_tx")
|
||||
logger.error(
|
||||
"%s pending_transaction %s was given up as failed but tx %s is in block %s — lottery paused for the operator",
|
||||
row.kind,
|
||||
row.id,
|
||||
txid,
|
||||
height,
|
||||
)
|
||||
|
||||
|
||||
async def _promote(session: AsyncSession, row: PendingTransaction) -> None:
|
||||
"""The tx did make it onto the chain before we died — finish what phase 2 of
|
||||
place_bet/request_withdrawal would have done."""
|
||||
@@ -190,8 +384,10 @@ async def _promote(session: AsyncSession, row: PendingTransaction) -> None:
|
||||
async def _abandon(session: AsyncSession, row: PendingTransaction, reason: str) -> None:
|
||||
"""The tx is gone for good. Release whatever it reserved so the funds come back,
|
||||
and roll the domain row back to something truthful."""
|
||||
reason = reason[:128]
|
||||
row.status = "failed"
|
||||
row.failure_reason = reason[:128]
|
||||
row.failure_reason = reason
|
||||
row.failed_at = datetime.now(timezone.utc) # B-78: starts the 14-day watch
|
||||
|
||||
released = await _release_inputs(session, row, await txids_of(session, row))
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from embit import script
|
||||
from sqlalchemy import select
|
||||
@@ -246,6 +247,7 @@ async def _release_failed_withdrawal(
|
||||
withdrawal.txid = None
|
||||
pending.status = "failed"
|
||||
pending.failure_reason = reason[:128]
|
||||
pending.failed_at = datetime.now(timezone.utc) # B-78 (DAT-36): starts the 14-day watch
|
||||
await recompute_balance(session, user_id)
|
||||
await write_audit_log(
|
||||
session,
|
||||
|
||||
Reference in new issue
Block a user