diff --git a/CLAUDE.md b/CLAUDE.md index a4af8b1..f18f722 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ The user communicates in Italian in chat — reply to them in Italian. Everythin ## Project status -The migration from FastAPI to Django is in progress on branch `django-migration` (plan: [roadmap.md](roadmap.md), steps A1–A10, B1–B6 done). The suite has 1288 tests: the old stack's under `tests/unit/`, the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Open defects of the old stack are in [BUGS.md](BUGS.md); accepted limits in [docs/limiti-noti.md](docs/limiti-noti.md); every closed `B-nn` is in [docs/storico-bug.md](docs/storico-bug.md) (`git log --all --grep 'B-nn'` finds its commit). +The migration from FastAPI to Django is in progress on branch `django-migration` (plan: [roadmap.md](roadmap.md), steps A1–A10, B1–B7 done). The suite has 1336 tests: the old stack's under `tests/unit/`, the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Open defects of the old stack are in [BUGS.md](BUGS.md); accepted limits in [docs/limiti-noti.md](docs/limiti-noti.md); every closed `B-nn` is in [docs/storico-bug.md](docs/storico-bug.md) (`git log --all --grep 'B-nn'` finds its commit). **[docs/](docs/README.md) is the specification, and it is authoritative.** Every verifiable rule there has a stable requirement ID (`RND-04`, `BET-02`, …) with `*Codice:*`/`*Test:*` pointers to what implements it; the index is [docs/come-funziona.md](docs/come-funziona.md) and the ID convention and change workflow are in [docs/processo-di-modifica.md](docs/processo-di-modifica.md). A behaviour change starts in the spec: edit or add the requirement, implement it, cite the ID in the test docstring and the commit message, and ship spec + code + test in one commit. If this file and the spec disagree, the spec wins and this file is the one to fix. Before writing code, read the relevant spec documents and the "Architecture" section below. diff --git a/README.md b/README.md index afbb43c..e280b84 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ python -m pytest # all tests python -m pytest tests/unit/test_hd.py # one file ``` -1288 unit tests cover HD derivation, PSBT building, the Electrum client, bets, +1336 unit tests cover HD derivation, PSBT building, the Electrum client, bets, deposits, withdrawals, the round/draw engine, RBF fee-bumping, admin config, the pending-inclusive balance calculation, and the SSE push channel. No automated integration tests against a live Electrum connection — mainnet diff --git a/apps/lottery/admin.py b/apps/lottery/admin.py index 7d273d5..b589a8a 100644 --- a/apps/lottery/admin.py +++ b/apps/lottery/admin.py @@ -83,7 +83,7 @@ class LotteryConfigAdmin(admin.ModelAdmin): commands, which ask for a reason (WEB-43). A change is audited with what changed.""" fields = ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds", - "draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb") + "draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb", "rbf_timeout_seconds") def has_add_permission(self, request): return False diff --git a/apps/lottery/bets.py b/apps/lottery/bets.py index be6c853..ca99895 100644 --- a/apps/lottery/bets.py +++ b/apps/lottery/bets.py @@ -77,6 +77,8 @@ def _sign_one(tx: ChainTransaction, account, fee_rate: int) -> bool: if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid is not None: return False # someone else got there first register_version(fresh, built.txid, built.raw_hex) + fresh.fee_rate_sat_vb = fee_rate # DAT-85: what the next bump starts from + fresh.save(update_fields=["fee_rate_sat_vb"]) bet.bet_txid, bet.bet_vout, bet.bet_amount_sats = built.txid, RECIPIENT_VOUT, built.recipient_sats bet.save(update_fields=["bet_txid", "bet_vout", "bet_amount_sats"]) audit.record("bet_signed", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, @@ -85,23 +87,23 @@ def _sign_one(tx: ChainTransaction, account, fee_rate: int) -> bool: return True -def _fail(tx: ChainTransaction, reason: str, *, release: bool) -> None: +def _fail(tx: ChainTransaction, reason: str, *, release: bool, action: str = "bet_refused") -> int: """Refused with proof (or unbuildable): the bet is rejected, the coins return, the movement stops counting.""" with transaction.atomic(): fresh = ChainTransaction.objects.select_for_update().get(pk=tx.pk) if fresh.status not in (ChainTransaction.Status.BUILDING, ChainTransaction.Status.PENDING): - return + return 0 fresh.status, fresh.failed_at, fresh.failure_reason = ChainTransaction.Status.FAILED, timezone.now(), reason[:500] fresh.save(update_fields=["status", "failed_at", "failure_reason"]) - if release: - reservation.release(fresh) + freed = reservation.release(fresh) if release else 0 bet = _bet_of(fresh) if bet is not None: bet.status = Participation.Status.REJECTED bet.save(update_fields=["status"]) Movement.objects.filter(participation=bet, kind=Movement.Kind.BET).update(status=Movement.Status.FAILED) - audit.record("bet_refused", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, - outcome="error", payload={"reason": reason[:500]}) + audit.record(action, actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, + outcome="error", payload={"reason": reason[:500], "coins_freed": freed}) + return freed def _settle(tx_id: int, txid: str, verdict, now: datetime) -> bool: @@ -111,8 +113,8 @@ def _settle(tx_id: int, txid: str, verdict, now: datetime) -> bool: if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid != txid: return False if verdict.outcome is Outcome.ACCEPTED: - fresh.status, fresh.broadcast_at = ChainTransaction.Status.PENDING, now - fresh.save(update_fields=["status", "broadcast_at"]) + fresh.status, fresh.broadcast_at, fresh.last_broadcast_at = ChainTransaction.Status.PENDING, now, now + fresh.save(update_fields=["status", "broadcast_at", "last_broadcast_at"]) audit.record("bet_broadcast", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, payload={"txid": txid, "note": verdict.reason}) return True diff --git a/apps/lottery/confirmation.py b/apps/lottery/confirmation.py new file mode 100644 index 0000000..92724f5 --- /dev/null +++ b/apps/lottery/confirmation.py @@ -0,0 +1,88 @@ +"""Confirmations of the platform's own transactions (roadmap B7, TX-30, TX-31, TX-26). + +Three phases like the deposit scan: read the pending transactions and every txid each has had; ask the servers with no +database open (a version in a block, with a merkle proof the others back); write the outcome, only for rows still +`pending`. The version that confirms is adopted even when it is not `current_txid`: an RBF replacement may not have won. +""" + +import asyncio +import logging +from datetime import datetime + +from django.db import transaction +from django.utils import timezone + +from . import audit +from .models import ChainTransaction, DepositAddress, Movement, Participation, Utxo + +logger = logging.getLogger(__name__) +WORKER = "worker" + + +def spending_address(tx: ChainTransaction) -> str | None: + """The address whose history shows the transaction: the player's for a bet or a withdrawal, the round's pool for a + payout (B8).""" + if tx.kind == ChainTransaction.Kind.PAYOUT: + return tx.round.pool_address if tx.round_id else None + owner = DepositAddress.objects.filter(user_id=tx.user_id).first() + return owner.address if owner else None + + +def plan() -> list[tuple[int, str, frozenset[str]]]: + wanted = [] + for tx in ChainTransaction.objects.filter(status=ChainTransaction.Status.PENDING).select_related("round").order_by("id"): + address = spending_address(tx) + txids = frozenset(tx.versions.values_list("txid", flat=True)) + if address and txids: + wanted.append((tx.pk, address, txids)) + return wanted + + +def apply_confirmations(confirmed: dict[int, tuple[str, int]], now: datetime | None = None) -> int: + now = now or timezone.now() + done = 0 + for tx_id, (txid, height) in confirmed.items(): + with transaction.atomic(): + fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id) + if fresh.status != ChainTransaction.Status.PENDING: + continue # TX-26: confirmed, abandoned or settled by someone else while the network was asked + version = fresh.versions.filter(txid=txid).first() + if version is None: + logger.error("transaction %s: confirmed txid %s is none of its versions", tx_id, txid) + continue + adopted = fresh.current_txid != txid + if adopted: # TX-31 + fresh.current_txid = txid + if version.raw_hex: + fresh.raw_hex = version.raw_hex + fresh.status, fresh.confirmed_at = ChainTransaction.Status.CONFIRMED, now + fresh.save(update_fields=["status", "confirmed_at", "current_txid", "raw_hex"]) + Utxo.objects.filter(reserved_by=fresh).update(spent_txid=txid) + if fresh.kind == ChainTransaction.Kind.BET: + _bet_confirmed(fresh, txid, now) + audit.record("tx_confirmed", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, + payload={"txid": txid, "height": height, "adopted": adopted, "kind": fresh.kind}) + done += 1 + return done + + +def _bet_confirmed(tx: ChainTransaction, txid: str, now: datetime) -> None: + """The bet takes part in the draw (BET, TX-07): found through its fixed links, never through the txid (TX-08).""" + movement = Movement.objects.filter(chain_transaction=tx, kind=Movement.Kind.BET).select_related("participation").first() + if movement is None or movement.participation is None: + return + bet = movement.participation + if bet.status == Participation.Status.PENDING: + bet.status, bet.confirmed_at, bet.bet_txid = Participation.Status.CONFIRMED, now, txid + bet.save(update_fields=["status", "confirmed_at", "bet_txid"]) + if movement.status == Movement.Status.PENDING: + movement.status, movement.confirmed_at = Movement.Status.CONFIRMED, now + movement.save(update_fields=["status", "confirmed_at"]) + + +def poll_confirmations(chain, now: datetime | None = None) -> int: + wanted = plan() + if not wanted: + return 0 + confirmed = asyncio.run(chain.observe_confirmations(wanted)) + return apply_confirmations(confirmed, now) if confirmed else 0 diff --git a/apps/lottery/electrum_chain.py b/apps/lottery/electrum_chain.py index 0bb634b..b2a894b 100644 --- a/apps/lottery/electrum_chain.py +++ b/apps/lottery/electrum_chain.py @@ -176,15 +176,20 @@ class ElectrumChain: if not self.majority(await self._ask_others(lists_it)): return False + return await self._proof_holds(active, entry.txid, entry.height) + + async def _proof_holds(self, active: ElectrumClient, txid: str, height: int) -> bool: + """ELC-26, TX-22: a header of the active server at `height` with valid work for the profile, and a merkle + proof from the others (by majority) that puts `txid` in that block.""" try: - header = await asyncio.wait_for(active.get_block_header(entry.height), ANSWER_TIMEOUT) + header = await asyncio.wait_for(active.get_block_header(height), ANSWER_TIMEOUT) except Exception: return False if not header_meets_its_own_target(str(header), self.profile.pow_limit): return False async def proves_it(client: ElectrumClient) -> bool: - return proof_matches_header(entry.txid, entry.height, await client.get_merkle(entry.txid, entry.height), str(header)) + return proof_matches_header(txid, height, await client.get_merkle(txid, height), str(header)) return self.majority(await self._ask_others(proves_it)) @@ -197,6 +202,70 @@ class ElectrumChain: return self.majority(await self._ask_others(lacks_it)) + # -- confirmations and presence (TX-30, TX-33) ---------------------------------------------------------------- + + @staticmethod + async def _history(client: ElectrumClient, address: str) -> list[tuple[str, int]] | None: + try: + raw = await asyncio.wait_for(client.get_history(address_to_scripthash(address)), ANSWER_TIMEOUT) + except Exception as error: + logger.warning("history of %s failed: %s", address, error) + return None + try: + return [(str(item["tx_hash"]), int(item["height"])) for item in raw] + except (KeyError, TypeError, ValueError): + return None + + async def observe_confirmations(self, wanted: list[tuple[int, str, frozenset[str]]]) -> dict[int, tuple[str, int]]: + """`wanted` is `(row id, spending address, every txid of the row)`. Returns, per row, the version that is in a + block *and* proven (TX-30): `{row id: (txid, height)}`. An address that cannot be read blocks nobody else.""" + active = await self._connect_active() + if active is None: + return {} + try: + histories: dict[str, list[tuple[str, int]] | None] = {} + confirmed: dict[int, tuple[str, int]] = {} + for row_id, address, txids in wanted: + if address not in histories: + histories[address] = await self._history(active, address) + history = histories[address] + if history is None: + continue + for txid, height in history: + if txid in txids and height > 0 and await self._proof_holds(active, txid, height): + confirmed[row_id] = (txid, height) + break + return confirmed + except Exception: + logger.exception("confirmation poll failed") + return {} + finally: + await active.close() + + async def versions_known(self, address: str, txids: frozenset[str]) -> tuple[set[str], bool]: + """TX-33: which of `txids` any server of the network has seen (mempool or block), and whether the network + answered enough for the *absence* of the rest to mean something (the active server and the quorum).""" + active = await self._connect_active() + if active is None: + return set(), False + try: + mine = await self._history(active, address) + + async def theirs(client: ElectrumClient): + history = await self._history(client, address) + return None if history is None else {txid for txid, _ in history if txid in txids} + + others = await self._ask_others(theirs) + known = ({txid for txid, _ in mine if txid in txids} if mine is not None else set()) + for answer in others: + if answer: + known |= answer + # an absence is a statement only if the active server answered and the others agree among themselves + complete = mine is not None and self.majority([a is not None for a in others]) + return known, complete + finally: + await active.close() + # -- broadcast (BET-32) ----------------------------------------------------------------------------------- async def transaction_known(self, active: ElectrumClient, txid: str) -> bool | None: diff --git a/apps/lottery/management/commands/run_worker.py b/apps/lottery/management/commands/run_worker.py index 4b021ae..217c4cb 100644 --- a/apps/lottery/management/commands/run_worker.py +++ b/apps/lottery/management/commands/run_worker.py @@ -42,13 +42,18 @@ def process_chain_bets() -> None: """BET-31/32: sign what the web admitted, send what was signed. On a real ledger only.""" from django.conf import settings - from apps.lottery import bets + from apps.lottery import bets, confirmation, rbf, reconcile from apps.lottery.electrum_chain import ElectrumChain chain = ElectrumChain(settings.ELECTRUM_ENDPOINTS, settings.PLM_PROFILE, settings.ELECTRUM_ALLOW_NO_QUORUM) done = bets.process_bets(chain) if any(done.values()): logger.info("bets: %s", done) + confirmed = confirmation.poll_confirmations(chain) # B7: TX-30 + bumped = rbf.bump_due(chain) # TX-32 + reconciled = reconcile.reconcile_once(chain) # TX-33 + if confirmed or bumped or any(reconciled.values()): + logger.info("transactions: confirmed=%s bumped=%s reconciled=%s", confirmed, bumped, reconciled) def tick() -> None: diff --git a/apps/lottery/migrations/0008_rbf_and_send_state.py b/apps/lottery/migrations/0008_rbf_and_send_state.py new file mode 100644 index 0000000..83f20fa --- /dev/null +++ b/apps/lottery/migrations/0008_rbf_and_send_state.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2.19 on 2026-11-16 15:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lottery', '0007_fee_rate'), + ] + + operations = [ + migrations.AddField( + model_name='chaintransaction', + name='attempt_count', + field=models.PositiveSmallIntegerField(default=1), + ), + migrations.AddField( + model_name='chaintransaction', + name='fee_rate_sat_vb', + field=models.PositiveIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='chaintransaction', + name='last_broadcast_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='lotteryconfig', + name='rbf_timeout_seconds', + field=models.PositiveIntegerField(default=900), + ), + migrations.AddConstraint( + model_name='lotteryconfig', + constraint=models.CheckConstraint(condition=models.Q(('rbf_timeout_seconds__gte', 60), ('rbf_timeout_seconds__lte', 86400)), name='lottery_config_rbf_timeout_range'), + ), + ] diff --git a/apps/lottery/models.py b/apps/lottery/models.py index df6a6c0..52567bf 100644 --- a/apps/lottery/models.py +++ b/apps/lottery/models.py @@ -37,6 +37,8 @@ class LotteryConfig(models.Model): max_round_pool_sats = models.BigIntegerField(default=0) # 0 = no cap # Network fee of the transactions built from now on, sat/vB (BET-04); a transaction in flight keeps its own. fee_rate_sat_vb = models.PositiveIntegerField(default=1) + # TX-32: how long a pending transaction waits before it is replaced with a higher fee. + rbf_timeout_seconds = models.PositiveIntegerField(default=900) # ADR-57: three distinct commands. pause_after_round = models.BooleanField(default=False) @@ -72,6 +74,9 @@ class LotteryConfig(models.Model): models.CheckConstraint( condition=Q(fee_rate_sat_vb__gte=1, fee_rate_sat_vb__lte=5000), name="lottery_config_fee_rate_range" ), + models.CheckConstraint( + condition=Q(rbf_timeout_seconds__gte=60, rbf_timeout_seconds__lte=86400), name="lottery_config_rbf_timeout_range" + ), ] @classmethod @@ -245,6 +250,10 @@ class ChainTransaction(models.Model): confirmed_at = models.DateTimeField(null=True, blank=True) failed_at = models.DateTimeField(null=True, blank=True) failure_reason = models.TextField(blank=True, default="") + # DAT-85: the current version's rate, the last send (RBF moves it; broadcast_at stays the first) and the tries. + fee_rate_sat_vb = models.PositiveIntegerField(null=True, blank=True) + last_broadcast_at = models.DateTimeField(null=True, blank=True) + attempt_count = models.PositiveSmallIntegerField(default=1) class Meta: indexes = [models.Index(fields=["status"])] diff --git a/apps/lottery/rbf.py b/apps/lottery/rbf.py new file mode 100644 index 0000000..6da3c91 --- /dev/null +++ b/apps/lottery/rbf.py @@ -0,0 +1,160 @@ +"""Replace-by-fee for the platform's transactions (roadmap B7, TX-32, TX-10..TX-13, TX-24). + +Same coins, same outputs to the counterpart; the increase comes out of one output only, the player's change. The +replacement is signed and **recorded as a version before it is sent**; after the send (accepted or uncertain) the row +moves to it with an update that matches only if the row is still `pending` and still on the txid that was replaced. +""" + +import asyncio +import logging +from datetime import datetime, timedelta + +from django.db import transaction +from django.utils import timezone +from embit import script +from embit.finalizer import finalize_psbt +from embit.psbt import PSBT +from embit.transaction import Transaction, TransactionInput, TransactionOutput +from plm_engine import hd +from plm_engine.psbt_builder import DUST_LIMIT_SATS, MAX_FEE_RATE_SAT_VB, RBF_SEQUENCE, estimate_vsize + +from . import audit, keys +from .chain import register_version +from .electrum_chain import Outcome +from .models import AuditEntry, ChainTransaction, DepositAddress, LotteryConfig, Participation + +logger = logging.getLogger(__name__) +WORKER = "worker" +FEE_RATE_INCREMENT = 1 # sat/vB more than the recorded rate, per bump (TX-10) +INCREMENTAL_RELAY_FEE_RATE = 1 # BIP125 rule 4, sat/vB + +REASON_MAX_RATE = "max_fee_rate" +REASON_NO_CHANGE = "no_change_output" +REASON_REFUSED = "replacement_refused" + + +class RbfError(Exception): + def __init__(self, message: str, reason: str): + super().__init__(message) + self.reason = reason + + +def due(now: datetime | None = None) -> list[ChainTransaction]: + """TX-32: pending bets whose last send is older than the timeout.""" + now = now or timezone.now() + limit = now - timedelta(seconds=LotteryConfig.get().rbf_timeout_seconds) + rows = ChainTransaction.objects.filter(kind=ChainTransaction.Kind.BET, status=ChainTransaction.Status.PENDING) + return [t for t in rows.order_by("id") if (t.last_broadcast_at or t.broadcast_at or t.created_at) <= limit] + + +def _build_replacement(tx: ChainTransaction, account) -> tuple[str, str, int, int]: + """Phase 2, no network and no open transaction: the signed replacement. Returns `(txid, raw hex, new rate, vsize)`.""" + if (tx.fee_rate_sat_vb or 1) >= MAX_FEE_RATE_SAT_VB: + raise RbfError(f"already at the maximum fee rate ({MAX_FEE_RATE_SAT_VB} sat/vB)", REASON_MAX_RATE) + owner = DepositAddress.objects.get(user=tx.user) + key = hd.user_key(account, owner.derivation_index) + own_script = script.p2wpkh(key.to_public()) + old = Transaction.parse(bytes.fromhex(tx.raw_hex)) + amounts = {(u.txid, u.vout): u.amount_sats for u in tx.reserved_utxos.all()} + input_amounts = [amounts[(v.txid.hex(), v.vout)] for v in old.vin] + old_fee = sum(input_amounts) - sum(o.value for o in old.vout) + vsize = estimate_vsize(len(old.vin), len(old.vout)) + rate = tx.fee_rate_sat_vb or 1 + target_rate = min(rate + FEE_RATE_INCREMENT, MAX_FEE_RATE_SAT_VB) + min_delta = vsize * INCREMENTAL_RELAY_FEE_RATE # BIP125: never below this, whatever the target says + delta = max(vsize * target_rate - old_fee, min_delta) + + change = next((i for i, o in enumerate(old.vout) if o.script_pubkey.data == own_script.data), None) + if change is None: + raise RbfError("there is no change output to take the increase from", REASON_NO_CHANGE) + available = old.vout[change].value + if available < min_delta: + raise RbfError(f"its change ({available} sat) cannot cover the minimum increase ({min_delta} sat)", REASON_NO_CHANGE) + outputs = list(old.vout) + if available - delta >= DUST_LIMIT_SATS: + outputs[change] = TransactionOutput(available - delta, outputs[change].script_pubkey) + else: # TX-11: nothing relayable would be left, so the output goes to the fee and this is the last replacement + del outputs[change] + delta = available + + new = Transaction(vin=[TransactionInput(v.txid, v.vout, sequence=RBF_SEQUENCE) for v in old.vin], vout=outputs) + psbt = PSBT(new) + for i, amount in enumerate(input_amounts): + psbt.inputs[i].witness_utxo = TransactionOutput(amount, own_script) + if psbt.sign_with(key) != len(old.vin): + raise RuntimeError("the replacement was not signed on every input") + final = finalize_psbt(psbt) + if final is None: + raise RuntimeError("failed to finalize the replacement") + new_rate = (old_fee + delta) // estimate_vsize(len(old.vin), len(outputs)) + return final.txid().hex(), final.serialize().hex(), new_rate, vsize + + +def _record_failure(tx: ChainTransaction, reason: str, message: str) -> None: + """TX-24: once per (txid, reason); the log carries every repeat.""" + seen = AuditEntry.objects.filter(action="rbf_bump_failed", subject_id=str(tx.pk)).values_list("payload", flat=True) + if any(p.get("txid") == tx.current_txid and p.get("reason") == reason for p in seen): + logger.info("bump of transaction %s failed again (%s): %s", tx.pk, reason, message) + return + audit.record("rbf_bump_failed", actor_label=WORKER, subject_type="chain_transaction", subject_id=tx.pk, + outcome=AuditEntry.Outcome.ERROR, reason=message, + payload={"txid": tx.current_txid, "reason": reason}) + + +def bump(tx_id: int, chain, now: datetime | None = None) -> str | None: + """Accelerate one transaction. Returns the new txid, or None if there was nothing to do (not pending any more, a + conflict, or an impossible/refused bump, which is recorded).""" + now = now or timezone.now() + tx = ChainTransaction.objects.select_related("user").filter(pk=tx_id).first() + if tx is None or tx.status != ChainTransaction.Status.PENDING: + return None + try: + account = keys.signing_account_key() + new_txid, raw_hex, new_rate, _ = _build_replacement(tx, account) + except RbfError as error: + _record_failure(tx, error.reason, str(error)) + return None + except keys.SigningKeyUnavailable as error: + logger.error("cannot bump transaction %s: %s", tx_id, error) + return None + old_txid = tx.current_txid + + with transaction.atomic(): # the candidate is recorded before it is sent (TX-12) + fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id) + if fresh.status != ChainTransaction.Status.PENDING or fresh.current_txid != old_txid: + return None + from .models import TransactionVersion + + if not TransactionVersion.objects.filter(txid=new_txid).exists(): + TransactionVersion.objects.create(transaction=fresh, txid=new_txid, raw_hex=raw_hex) + + verdict = asyncio.run(chain.broadcast(raw_hex, new_txid)) + if verdict.outcome is Outcome.REFUSED: # the candidate stays recorded; nobody will confirm it (TX-24) + _record_failure(tx, REASON_REFUSED, f"the network refused the replacement: {verdict.reason}") + return None + + with transaction.atomic(): # TX-12: move the row, only if it is still exactly what was replaced + moved = ChainTransaction.objects.filter( + pk=tx_id, status=ChainTransaction.Status.PENDING, current_txid=old_txid + ).update(current_txid=new_txid, raw_hex=raw_hex, fee_rate_sat_vb=new_rate, last_broadcast_at=now, + attempt_count=tx.attempt_count + 1) + if not moved: # TX-13: it changed under us; the candidate stays and confirmation looks for it anyway + logger.warning("transaction %s changed while its replacement was being sent: conflict", tx_id) + return None + Participation.objects.filter(movements__chain_transaction_id=tx_id, bet_txid=old_txid).update(bet_txid=new_txid) + audit.record("rbf_bumped", actor_label=WORKER, subject_type="chain_transaction", subject_id=tx_id, + payload={"from": old_txid, "to": new_txid, "fee_rate_sat_vb": new_rate, + "network": verdict.outcome.value}) + return new_txid + + +def bump_due(chain, now: datetime | None = None) -> int: + """TX-32: accelerate every transaction that waited too long. An error on one row stops no other.""" + count = 0 + for tx in due(now): + try: + if bump(tx.pk, chain, now): + count += 1 + except Exception: + logger.exception("bumping transaction %s failed", tx.pk) + return count diff --git a/apps/lottery/reconcile.py b/apps/lottery/reconcile.py new file mode 100644 index 0000000..06712c5 --- /dev/null +++ b/apps/lottery/reconcile.py @@ -0,0 +1,125 @@ +"""Reconciling what the database believes with what the network knows (roadmap B7, TX-33, TX-15..TX-17, TX-23, TX-26). + +A row is only ever given up on proof (ADR-51): a refusal of the newest version for missing or already spent inputs +while no server knows any version of it. A timeout, silence or a refusal for the fee is never a proof. The second +proof of TX-17 (an input spent in a block by another transaction) arrives with the reorg work of B10. +""" + +import asyncio +import logging +from datetime import datetime, timedelta + +from django.db import transaction +from django.utils import timezone + +from . import audit +from .bets import WORKER, _fail +from .confirmation import apply_confirmations, spending_address +from .electrum_chain import Outcome +from .models import ChainTransaction, LotteryConfig + +logger = logging.getLogger(__name__) + +GRACE_BUILDING = timedelta(seconds=120) +GRACE_PENDING = timedelta(hours=6) +FAILED_WATCH = timedelta(days=14) +# Wordings by which a node says the inputs are not there to spend. Anything else is not a proof (TX-17). +_MISSING_INPUTS = ("missing-inputs", "missing inputs", "missingorspent", "already spent") + + +def is_missing_inputs(reason: str) -> bool: + lowered = reason.lower() + return any(marker in lowered for marker in _MISSING_INPUTS) + + +def _is_due(tx: ChainTransaction, now: datetime) -> bool: + if tx.status == ChainTransaction.Status.BUILDING: + return tx.current_txid is not None and now - tx.created_at >= GRACE_BUILDING + return now - (tx.broadcast_at or tx.created_at) >= GRACE_PENDING + + +def _promote(tx_id: int, now: datetime, txid: str) -> bool: + """TX-16: the network has it. A `building` row becomes `pending` (so confirmation and RBF can take over).""" + with transaction.atomic(): + fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id) + if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid != txid: # TX-26 + return False + fresh.status, fresh.broadcast_at, fresh.last_broadcast_at = ChainTransaction.Status.PENDING, now, now + fresh.save(update_fields=["status", "broadcast_at", "last_broadcast_at"]) + audit.record("pending_tx_recovered", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, + payload={"txid": txid}) + return True + + +def reconcile_once(chain, now: datetime | None = None) -> dict[str, int]: + now = now or timezone.now() + done = {"recovered": 0, "abandoned": 0, "resurrected": 0} + rows = ChainTransaction.objects.filter( + status__in=[ChainTransaction.Status.BUILDING, ChainTransaction.Status.PENDING] + ).select_related("round").order_by("id") + for tx in rows: + if not _is_due(tx, now): + continue + try: + _reconcile_row(tx, chain, now, done) + except Exception: + logger.exception("reconciling transaction %s failed", tx.pk) + _watch_failed(chain, now, done) + return done + + +def _reconcile_row(tx: ChainTransaction, chain, now: datetime, done: dict[str, int]) -> None: + address = spending_address(tx) + txids = frozenset(tx.versions.values_list("txid", flat=True)) + if not address or not txids: + return + known, complete = asyncio.run(chain.versions_known(address, txids)) + if known: # TX-15: at least one server has at least one version: it is alive + if _promote(tx.pk, now, tx.current_txid): + done["recovered"] += 1 + return + if not complete or not tx.raw_hex: # an unreadable network proves nothing + return + verdict = asyncio.run(chain.broadcast(tx.raw_hex, tx.current_txid)) + if verdict.outcome is Outcome.ACCEPTED: + if _promote(tx.pk, now, tx.current_txid): + done["recovered"] += 1 + elif verdict.outcome is Outcome.REFUSED: + if is_missing_inputs(verdict.reason): # TX-17, proof 2 + _fail(tx, f"abandoned: {verdict.reason}", release=True, action="tx_abandoned") + done["abandoned"] += 1 + elif _promote(tx.pk, now, tx.current_txid): # refused for the fee: not a proof; let RBF accelerate it + done["recovered"] += 1 + + +def _watch_failed(chain, now: datetime, done: dict[str, int]) -> None: + """TX-23: a transaction given up for dead can still confirm. If one does, say it once and stop new bets.""" + rows = ChainTransaction.objects.filter( + status=ChainTransaction.Status.FAILED, failed_at__gte=now - FAILED_WATCH + ).select_related("round") + wanted = [] + for tx in rows: + if audit_exists("tx_resurrected", tx.pk): + continue + address, txids = spending_address(tx), frozenset(tx.versions.values_list("txid", flat=True)) + if address and txids: + wanted.append((tx.pk, address, txids)) + if not wanted: + return + for tx_id, (txid, height) in asyncio.run(chain.observe_confirmations(wanted)).items(): + with transaction.atomic(): + if audit_exists("tx_resurrected", tx_id): + continue + audit.record("tx_resurrected", actor_label=WORKER, subject_type="chain_transaction", subject_id=tx_id, + outcome="error", payload={"txid": txid, "height": height}) + LotteryConfig.objects.filter(pk=1).update(new_bets_suspended=True) # the operator puts things in order + done["resurrected"] += 1 + + +def audit_exists(action: str, subject_id: int) -> bool: + from .models import AuditEntry + + return AuditEntry.objects.filter(action=action, subject_id=str(subject_id)).exists() + + +__all__ = ["apply_confirmations", "is_missing_inputs", "reconcile_once"] diff --git a/docs/legacy-fastapi-notes.md b/docs/legacy-fastapi-notes.md index f6dfec6..3fb9b08 100644 --- a/docs/legacy-fastapi-notes.md +++ b/docs/legacy-fastapi-notes.md @@ -4,7 +4,7 @@ ## Project status (old stack) -All 10 stages of the original build order are code-complete and unit-tested — 1288 tests, the old stack's under `tests/unit/` and the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Beyond them: Docker + Caddy deployment, admin dashboard (`/admin`), static test UI (`/`), pending-inclusive balance display, an SSE push channel layered over the original polling, self-service password change + admin password reset, and the reconciliation/corroboration machinery below. +All 10 stages of the original build order are code-complete and unit-tested — 1336 tests, the old stack's under `tests/unit/` and the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Beyond them: Docker + Caddy deployment, admin dashboard (`/admin`), static test UI (`/`), pending-inclusive balance display, an SSE push channel layered over the original polling, self-service password change + admin password reset, and the reconciliation/corroboration machinery below. Verified on mainnet with real money: registration + address derivation, deposit crediting (at the time, 1-conf; deposits now need `deposit_confirmations`), a real 10 PLM bet (broadcast → confirmed → change credited back), and one full round cycle (close → draw on a real block hash → 70/30 payout with sat math checked against the broadcast tx → confirmation → close → next round auto-opened). **Withdrawal and the RBF bump path have never been exercised against a live broadcast** — unit-tested only. @@ -30,7 +30,7 @@ PYTHONPATH=. python scripts/decrypt_master_key.py # ops recovery: decrypt+pr PYTHONPATH=. python scripts/encrypt_master_key.py # ops bootstrap: import an externally-generated xprv (--overwrite to replace) PYTHONPATH=. python scripts/electrum_smoke_test.py # manual check: connect, handshake, subscribe to headers, print the tip -python -m pytest # all 1288 tests +python -m pytest # all 1336 tests python -m pytest tests/unit/test_hd.py # one file python -m pytest tests/unit/test_hd.py::test_derivation_is_deterministic # one test ``` diff --git a/docs/modello-dati.md b/docs/modello-dati.md index 6dfa4d6..33301ab 100644 --- a/docs/modello-dati.md +++ b/docs/modello-dati.md @@ -1500,7 +1500,7 @@ Le tabelle di catena (UTXO, versioni delle transazioni, depositi, indici HD) son si elimina. Ogni importo è un intero in unità minime ([DAT-01]) e ogni timestamp è in UTC con fuso ([DAT-02]). *Stato:* implementato (A2) · *Codice:* `apps/lottery/models.py` · *Test:* `tests/django/test_models.py::test_rows_that_carry_money_cannot_be_deleted_with_their_owner` - **[DAT-77]** **Parametri e pause: `LotteryConfig`.** Una sola riga (`id = 1`, vincolo) con quota (10 PLM), - durata (600 s), pausa fra round (30 s), conferme richieste a un deposito (1–100, default 6, [DAT-62]; si cambia da `/admin/` con audit e vale subito per gli accrediti successivi), conferme del blocco di estrazione (1–20, default 3), commissione di rete in sat/vB (1–5 000, default 1: vale per le transazioni costruite dopo), tetto del + durata (600 s), pausa fra round (30 s), conferme richieste a un deposito (1–100, default 6, [DAT-62]; si cambia da `/admin/` con audit e vale subito per gli accrediti successivi), tempo oltre il quale una transazione lenta si accelera (`rbf_timeout_seconds`, 60–86 400 s, default 900), conferme del blocco di estrazione (1–20, default 3), commissione di rete in sat/vB (1–5 000, default 1: vale per le transazioni costruite dopo), tetto del montepremi (0 = nessuno) e i tre interruttori di [ADR-57](adr/ADR-57-due-pause-distinte-dopo-round-e-incidente.md): `pause_after_round`, `new_bets_suspended`, `withdrawals_suspended`. Valori non positivi o fuori intervallo sono rifiutati dal database. I parametri si **copiano su `Round` all'apertura** ([DAT-61]); la copia la fa il @@ -1557,4 +1557,8 @@ Le tabelle di catena (UTXO, versioni delle transazioni, depositi, indici HD) son Un output è un solo deposito e un `credited` ha sempre il movimento e la data di accredito. *Stato:* tabella implementata (B4); il listener e l'accredito sono B5 · *Codice:* `apps/lottery/models.py:Deposit` · *Test:* `tests/django/test_chain_models.py::test_a_credited_deposit_has_its_movement_and_date`, `test_an_output_is_one_deposit` +- **[DAT-85]** **Stato di invio di una transazione.** `ChainTransaction` registra la commissione della versione corrente + (`fee_rate_sat_vb`), l'istante dell'ultimo invio (`last_broadcast_at`, che l'RBF aggiorna mentre `broadcast_at` resta il primo) e il numero di + tentativi (`attempt_count`). Sono ciò che il bumper e il riconciliatore leggono ([TX-32], [TX-33]). + *Stato:* implementato (B7) · *Codice:* `apps/lottery/models.py:ChainTransaction` · *Test:* `tests/django/test_chain_rbf.py` diff --git a/docs/transazioni-e-riconciliazione.md b/docs/transazioni-e-riconciliazione.md index 63b4655..8a423d9 100644 --- a/docs/transazioni-e-riconciliazione.md +++ b/docs/transazioni-e-riconciliazione.md @@ -420,3 +420,42 @@ Nel pannello `/admin` (vedi [guida-admin.md](guida-admin.md) e correzioni in corso anche `rbf_bump_failed` ([TX-24]) e `pending_tx_resurrected` ([TX-23]); - **Round** — stato di ogni round, vincitore e txid del pagamento. + +## Transazioni sul sito Django (B7) + +Il sito Django riscrive conferma, accelerazione e riconciliazione sull'ORM (`ChainTransaction`, [DAT-83]). Per ora riguardano le +**scommesse**; prelievi e pagamenti (B8, B9) usano gli stessi meccanismi. Restano fuori, e sono B10: il rilevamento delle +riorganizzazioni (finestra degli ultimi header, [ELC-22]…[ELC-24]) e le monete già accreditate che poi spariscono ([DEP-06]…[DEP-08]). + +- **[TX-30]** **Conferma verificata, con tutte le versioni.** Il worker legge la cronologia dell'indirizzo che spende ogni transazione + `pending` e cerca **ogni** versione ([DAT-83]); una versione con altezza maggiore di 0 è candidata e si confermerà solo con la prova + merkle di [TX-22] (header dell'altezza con lavoro valido per il profilo, prova data dagli altri server a maggioranza, + [ELC-14]). Senza prova la riga resta `pending` e si riprova. Tre fasi: lettura del database, rete senza database aperto, scrittura. + *Stato:* implementato (B7) · *Codice:* `apps/lottery/confirmation.py`, `apps/lottery/electrum_chain.py:ElectrumChain.observe_confirmations` · + *Test:* `tests/django/test_chain_confirmation.py`, `tests/integration/test_confirmation_regtest.py` +- **[TX-31]** **Si adotta la versione confermata.** Se a confermarsi è una versione **diversa** da `current_txid` (l'originale mentre + una sostituzione era in volo, o una sostituzione che il processo non aveva fatto in tempo a registrare), la riga adotta quel txid e quella + transazione grezza, e lo propaga a scommessa (`bet_txid`) e monete spese (`spent_txid`). Alla conferma di una scommessa: la + `Participation` diventa `confirmed` con `confirmed_at` ed entra nell'estrazione, il movimento di puntata diventa `confirmed`, le monete + spese risultano spese, e l'audit scrive `tx_confirmed`; tutto nella stessa transazione del database ([TX-26]). + *Stato:* implementato (B7) · *Codice:* `apps/lottery/confirmation.py:apply_confirmations` · + *Test:* `tests/django/test_chain_confirmation.py::test_a_bet_is_confirmed_and_enters_the_draw`, `test_an_earlier_version_that_confirms_is_adopted` +- **[TX-32]** **Accelerazione (RBF).** Una transazione `pending` il cui ultimo invio è più vecchio di `rbf_timeout_seconds` + (`LotteryConfig`, 900 s) è ricostruita con una commissione di 1 sat/vB in più (mai oltre 5 000, mai meno dell'aumento minimo di BIP125), + spendendo le stesse monete; l'aumento si toglie **solo** dal resto del giocatore, e se ne resterebbe meno di 294 sat il resto sparisce + e quella è l'ultima accelerazione ([TX-10], [TX-11]). La sostituzione firmata è registrata come versione **prima** di trasmetterla + ([TX-12]); dopo l'invio (accettato o incerto) la riga passa alla nuova versione con un aggiornamento condizionato a stato `pending` e + `current_txid` invariato, altrimenti è un conflitto e la candidata resta registrata ([TX-13]). Se la rete rifiuta la sostituzione o + l'accelerazione è impossibile, la riga non cambia e si scrive `rbf_bump_failed` una sola volta per coppia (txid, motivo) ([TX-24]). + *Stato:* implementato (B7) · *Codice:* `apps/lottery/rbf.py` · + *Test:* `tests/django/test_chain_rbf.py`, `tests/integration/test_confirmation_regtest.py` +- **[TX-33]** **Riconciliazione e abbandono solo con prova.** Il worker esamina le transazioni `building` da oltre 120 s e `pending` da oltre + 6 ore dal primo invio. Se almeno un server conosce una versione, la riga è viva: `building` passa a `pending` (`pending_tx_recovered`). + Se nessuno la conosce, si **ritrasmette** la versione più recente: accettata = viva; rifiutata con prova **per ingressi mancanti o già spesi** + = abbandono (la transazione è `failed` con il motivo, le monete ancora impegnate da *questa* transazione tornano libere, la scommessa è + respinta, il movimento non conta più, `tx_abandoned`); rifiutata per altro (commissione) o incerta = nessun abbandono. Le righe `failed` + restano sorvegliate 14 giorni: se una loro versione risulta confermata si scrive una sola volta `tx_resurrected` e le nuove puntate + si sospendono per l'operatore ([TX-23]). Non esiste un tempo massimo che abbandoni una transazione che un server tiene in mempool. + *Stato:* implementato (B7); la seconda prova di [TX-17] (ingresso speso in un blocco da un'altra transazione) è B10 · + *Codice:* `apps/lottery/reconcile.py` · *Test:* `tests/django/test_chain_reconcile.py` + diff --git a/roadmap.md b/roadmap.md index c71191f..e0a784e 100644 --- a/roadmap.md +++ b/roadmap.md @@ -45,10 +45,10 @@ Uscita: flusso completo su regtest; un riavvio durante il broadcast, due richies - [x] **B4 · Modelli di catena.** UTXO, transazioni con tutte le versioni TXID, depositi, assegnazione dell'indice HD; la chiave di firma è accessibile solo al processo worker. *Requisiti:* DAT, TX. *Test:* vincoli, derivazione identica a parità di input. - [x] **B5 · Listener e depositi reali.** Listener Electrum sull'ORM con rotazione, header validati, quorum e soglie di conferma al posto del deposito simulato. *Requisiti:* DEP, ELC. *Test:* unit con server finto + integrazione su regtest. - [x] **B6 · Prenotazione UTXO e puntata reale.** Prenotazione atomica degli UTXO nel database, identificativo idempotente, «salva operazione e transazione firmata prima del broadcast», verdetto del broadcast (accettato, rifiutato, incerto). *Requisiti:* BET, TX-26. *Test:* due richieste concorrenti non spendono gli stessi input; timeout ambiguo riconciliato. -- [ ] **B7 · Conferme, riconciliazione e RBF.** Poller, riconciliatore e bumper sull'ORM con tutte le versioni TXID e adozione della versione confermata. *Requisiti:* TX. *Test:* conferma di una versione RBF precedente, ripresa dopo riavvio. +- [x] **B7 · Conferme, riconciliazione e RBF.** Poller, riconciliatore e bumper sull'ORM con tutte le versioni TXID e adozione della versione confermata. *Requisiti:* TX. *Test:* conferma di una versione RBF precedente, ripresa dopo riavvio. - [ ] **B8 · Estrazione e pagamento reali.** Commit-reveal con blocco a quorum, verifica del montepremi, pagamento 70/30 con riserva per l'RBF, pausa automatica sui casi bloccanti. *Requisiti:* RND. *Test:* round completo su regtest, riorganizzazione simulata. - [ ] **B9 · Prelievi e storico.** Prelievo con indirizzo, importo, commissione e netto previsti; storico con stato dell'operazione e TXID corrente con link all'explorer. *Requisiti:* WDR, WEB-24. *Test:* prelievo su regtest, sospensione dei prelievi. -- [ ] **B10 · Robustezza e chiusura di fase.** Test di riavvio durante il broadcast, richieste concorrenti web/worker, riorganizzazione; sostituzione definitiva dei blocchi in memoria. *Requisiti:* ARC-70…72, ADR-55. +- [ ] **B10 · Robustezza e chiusura di fase.** Test di riavvio durante il broadcast, richieste concorrenti web/worker, riorganizzazione; sostituzione definitiva dei blocchi in memoria. **Eredità di B5 e B7:** finestra degli ultimi header con rilevamento delle riorganizzazioni e pausa su quelle profonde (ELC-22…24), monete già accreditate che spariscono, spese altrove o reintegrate (DEP-06…08) e la seconda prova di abbandono di TX-17 (ingresso speso in un blocco da un'altra transazione). *Requisiti:* ARC-70…72, ADR-55. - [ ] **B11 · Prova guidata del round su regtest.** Script `scripts/` che, su un database nuovo, crea giocatori con fondi, li fa puntare, fa scadere il round e **mina i blocchi su regtest** (B3) invece di aspettare il tempo; controlla chiusura, estrazione, pagamento e nuovo round, la conservazione dei fondi (vincita 70% più commissione 30% uguale al montepremi), l'ordine dell'audit e che la pagina pubblica `/risultati//` ricalcoli il vincitore senza nomi utente. Casi: quattro giocatori, un solo giocatore, puntata rifiutata, riavvio del worker durante l'estrazione. *Requisiti:* RND, WEB-12, WEB-13. *Test:* lo script esce con errore se un controllo fallisce; la prova fatta a mano con la catena simulata (4 giocatori, 40 PLM, 28 al vincitore, 12 alla piattaforma) è il riferimento. ## Fase C — Verifiche prima dell'uso pubblico diff --git a/tests/django/test_chain_confirmation.py b/tests/django/test_chain_confirmation.py new file mode 100644 index 0000000..4358622 --- /dev/null +++ b/tests/django/test_chain_confirmation.py @@ -0,0 +1,176 @@ +"""B7 — confirmations: TX-30, TX-31, TX-26, TX-08.""" + +import struct +from datetime import timedelta + +import pytest +from django.utils import timezone +from embit.transaction import Transaction +from plm_engine.electrum.client import ElectrumEndpoint +from plm_engine.headers import header_meets_its_own_target +from plm_engine.network import REGTEST + +from apps.lottery import bets, chain, confirmation +from apps.lottery.electrum_chain import ElectrumChain +from apps.lottery.models import AuditEntry, ChainTransaction, Movement, Participation, Utxo +from tests.django.test_chain_bets import ScriptedChain, bet, make_round, placed_and_signed, player, world # noqa: F401 + +pytestmark = pytest.mark.django_db + + +def pending(world): + user, round_, tx = placed_and_signed(world) + bets.broadcast_pending(ScriptedChain()) + tx.refresh_from_db() + return user, round_, tx + + +# --- the write phase ------------------------------------------------------------------------------------------ + + +def test_a_bet_is_confirmed_and_enters_the_draw(world): + user, round_, tx = pending(world) + assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 1 + tx.refresh_from_db() + bet_row = Participation.objects.get() + assert tx.status == "confirmed" and tx.confirmed_at + assert (bet_row.status, bet_row.bet_txid) == ("confirmed", tx.current_txid) and bet_row.confirmed_at + assert Movement.objects.get(kind="bet").status == "confirmed" + assert set(Utxo.objects.filter(reserved_by=tx).values_list("spent_txid", flat=True)) == {tx.current_txid} + entry = AuditEntry.objects.get(action="tx_confirmed") + assert entry.payload["height"] == 120 and entry.payload["adopted"] is False + + +def test_an_earlier_version_that_confirms_is_adopted(world): + """TX-31: the original won while a replacement was in flight.""" + user, round_, tx = pending(world) + original = tx.current_txid + replacement = "ee" * 32 + chain.register_version(tx, replacement, "00aa") # the row follows the replacement... + Participation.objects.update(bet_txid=replacement) + assert confirmation.apply_confirmations({tx.pk: (original, 130)}) == 1 # ...but the original is what the chain mined + tx.refresh_from_db() + assert tx.current_txid == original and tx.raw_hex and tx.raw_hex != "00aa" + assert Participation.objects.get().bet_txid == original + assert Utxo.objects.get(reserved_by=tx).spent_txid == original + assert AuditEntry.objects.get(action="tx_confirmed").payload["adopted"] is True + + +def test_a_row_that_is_no_longer_pending_is_left_alone(world): + user, round_, tx = pending(world) + ChainTransaction.objects.filter(pk=tx.pk).update(status="failed", failed_at=timezone.now(), failure_reason="x") + assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 0 + assert Participation.objects.get().status == "pending" + + +def test_confirming_twice_changes_nothing_the_second_time(world): + user, round_, tx = pending(world) + confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) + assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 0 + assert AuditEntry.objects.filter(action="tx_confirmed").count() == 1 + + +def test_the_plan_lists_pending_rows_with_every_version(world): + user, round_, tx = pending(world) + chain.register_version(tx, "ee" * 32, "00aa") + (row,) = confirmation.plan() + assert row[0] == tx.pk and row[1] == chain.assign_deposit_address(user).address and len(row[2]) == 2 + + +def test_a_building_row_is_not_polled(world): + user, round_, tx = placed_and_signed(world) # signed, never accepted: still building + assert confirmation.plan() == [] + + +# --- the network phase -------------------------------------------------------------------------------------------- + + +def mined(root: bytes) -> str: + for nonce in range(10_000): + header = (struct.pack("= estimate_vsize(1, 2) # BIP125: at least 1 sat/vB more + assert (tx.fee_rate_sat_vb, tx.attempt_count) == (2, 2) and tx.last_broadcast_at > timezone.now() - timedelta(seconds=30) + assert tx.versions.count() == 2 and Participation.objects.get().bet_txid == new_txid + assert AuditEntry.objects.filter(action="rbf_bumped").count() == 1 + + +def test_the_replacement_is_recorded_before_it_is_sent(world): + user, round_, tx = pending(world) + old_txid = tx.current_txid + age(tx, 1000) + + class Crashing: + async def broadcast(self, raw, txid): + raise RuntimeError("the process died here") + + with pytest.raises(RuntimeError): + rbf.bump(tx.pk, Crashing()) + tx.refresh_from_db() + assert tx.current_txid == old_txid and tx.versions.count() == 2 # the candidate is known, the row not moved + + +def test_an_uncertain_send_still_moves_the_row(world): + """TX-12: the replacement may well be on the network, so the row follows it.""" + user, round_, tx = pending(world) + age(tx, 1000) + assert rbf.bump(tx.pk, ScriptedChain(Verdict(Outcome.UNCERTAIN, "timeout"))) is not None + tx.refresh_from_db() + assert tx.attempt_count == 2 and tx.versions.count() == 2 + + +def test_a_refused_replacement_leaves_the_row_and_is_recorded_once(world): + user, round_, tx = pending(world) + old_txid = tx.current_txid + age(tx, 1000) + refusal = Verdict(Outcome.REFUSED, "insufficient fee") + assert rbf.bump(tx.pk, ScriptedChain(refusal)) is None + assert rbf.bump(tx.pk, ScriptedChain(refusal)) is None + tx.refresh_from_db() + assert tx.current_txid == old_txid and tx.status == "pending" + assert AuditEntry.objects.filter(action="rbf_bump_failed").count() == 1 # TX-24: once per (txid, reason) + + +def test_a_row_that_changed_meanwhile_is_a_conflict_not_an_overwrite(world, monkeypatch): + """TX-13: confirmed while the replacement was being sent. The row is not overwritten, the candidate stays.""" + import asyncio + + user, round_, tx = pending(world) + age(tx, 1000) + real_run = asyncio.run + + def run_then_confirm(coroutine): + result = real_run(coroutine) # the send happens... + ChainTransaction.objects.filter(pk=tx.pk).update(status="confirmed", confirmed_at=timezone.now()) # ...then this + return result + + monkeypatch.setattr(rbf.asyncio, "run", run_then_confirm) + assert rbf.bump(tx.pk, ScriptedChain()) is None + tx.refresh_from_db() + assert tx.status == "confirmed" and tx.attempt_count == 1 and tx.versions.count() == 2 + + +def test_a_transaction_that_is_no_longer_pending_is_skipped(world): + user, round_, tx = pending(world) + ChainTransaction.objects.filter(pk=tx.pk).update(status="confirmed", confirmed_at=timezone.now()) + assert rbf.bump(tx.pk, ScriptedChain()) is None and tx.versions.count() == 1 + + +# --- limits ------------------------------------------------------------------------------------------------------ + + +def test_the_maximum_fee_rate_is_never_exceeded(world): + user, round_, tx = pending(world) + ChainTransaction.objects.filter(pk=tx.pk).update(fee_rate_sat_vb=MAX_FEE_RATE_SAT_VB) + assert rbf.bump(tx.pk, ScriptedChain()) is None + assert AuditEntry.objects.get(action="rbf_bump_failed").payload["reason"] == "max_fee_rate" + + +def test_a_change_that_would_fall_below_dust_is_dropped_and_that_is_the_last_bump(world): + user, round_, tx = pending(world, funds=BET + 300) # a change of 300 sat + age(tx, 1000) + assert rbf.bump(tx.pk, ScriptedChain()) is not None + tx.refresh_from_db() + assert len(Transaction.parse(bytes.fromhex(tx.raw_hex)).vout) == 1 # only the pool output is left + age(tx, 1000) + assert rbf.bump(tx.pk, ScriptedChain()) is None # nothing left to take the increase from + assert AuditEntry.objects.get(action="rbf_bump_failed").payload["reason"] == "no_change_output" + + +def test_an_error_on_one_row_does_not_stop_the_others(world): + user, round_, first = pending(world) + other, other_round = player("luigi"), round_ + bet(other, other_round) + bets.sign_pending() + bets.broadcast_pending(ScriptedChain()) + for t in ChainTransaction.objects.all(): + age(t, 1000) + ChainTransaction.objects.filter(pk=first.pk).update(fee_rate_sat_vb=MAX_FEE_RATE_SAT_VB) # this one cannot + assert rbf.bump_due(ScriptedChain()) == 1 + + +def test_without_the_signing_key_nothing_is_bumped(world): + user, round_, tx = pending(world) + age(tx, 1000) + world.PROCESS_ROLE = "web" + assert rbf.bump(tx.pk, ScriptedChain()) is None + tx.refresh_from_db() + assert tx.versions.count() == 1 and tx.attempt_count == 1 diff --git a/tests/django/test_chain_reconcile.py b/tests/django/test_chain_reconcile.py new file mode 100644 index 0000000..509e4f9 --- /dev/null +++ b/tests/django/test_chain_reconcile.py @@ -0,0 +1,166 @@ +"""B7 — reconciliation: TX-33, TX-15..TX-17, TX-23, TX-26.""" + +from datetime import timedelta + +import pytest +from django.utils import timezone + +from apps.lottery import bets, reconcile, reservation +from apps.lottery.balances import balance_of +from apps.lottery.electrum_chain import Outcome, Verdict +from apps.lottery.models import AuditEntry, ChainTransaction, LotteryConfig, Movement, Participation, Utxo +from tests.django.test_chain_bets import BET, ScriptedChain, bet, make_round, placed_and_signed, player, world # noqa: F401 + +pytestmark = pytest.mark.django_db + + +class Net: + """The network as the reconciler asks about it.""" + + def __init__(self, known=(), complete=True, verdict=None, confirmed=None): + self.known, self.complete, self.verdict, self.confirmed = set(known), complete, verdict, confirmed or {} + self.sent = [] + + async def versions_known(self, address, txids): + return {t for t in txids if t in self.known}, self.complete + + async def broadcast(self, raw, txid): + self.sent.append(txid) + return self.verdict or Verdict(Outcome.UNCERTAIN, "no answer") + + async def observe_confirmations(self, wanted): + return {row: self.confirmed[row] for row, _a, _t in wanted if row in self.confirmed} + + +def old(tx, *, building_for=300, pending_for=None): + stamp = timezone.now() - timedelta(seconds=building_for) + ChainTransaction.objects.filter(pk=tx.pk).update(created_at=stamp) + if pending_for: + ChainTransaction.objects.filter(pk=tx.pk).update(broadcast_at=timezone.now() - timedelta(seconds=pending_for)) + + +def building(world): + user, round_, tx = placed_and_signed(world) + old(tx) + return user, round_, tx + + +MISSING = "bad-txns-inputs-missingorspent" + + +# --- presence ----------------------------------------------------------------------------------------------------- + + +def test_a_row_the_network_knows_is_promoted_to_pending(world): + user, round_, tx = building(world) + assert reconcile.reconcile_once(Net(known={tx.current_txid}))["recovered"] == 1 + tx.refresh_from_db() + assert tx.status == "pending" and tx.broadcast_at + assert AuditEntry.objects.filter(action="pending_tx_recovered").count() == 1 + + +def test_a_known_non_current_version_also_means_alive(world): + """TX-15: any version known to any server is enough, not only the current one.""" + from apps.lottery.chain import register_version + + user, round_, tx = building(world) + earlier = tx.current_txid + register_version(tx, "ee" * 32, "00aa") # the row now follows a replacement the network has not seen + assert reconcile.reconcile_once(Net(known={earlier}))["recovered"] == 1 + assert ChainTransaction.objects.get().status == "pending" + + +def test_a_fresh_building_row_is_not_examined_yet(world): + user, round_, tx = placed_and_signed(world) # created just now: inside the 120 s grace + net = Net(known=set()) + assert reconcile.reconcile_once(net) == {"recovered": 0, "abandoned": 0, "resurrected": 0} and net.sent == [] + + +def test_a_pending_row_is_examined_only_after_six_hours(world): + user, round_, tx = building(world) + reconcile._promote(tx.pk, timezone.now(), tx.current_txid) + old(tx, pending_for=5 * 3600) + net = Net() + reconcile.reconcile_once(net) + assert net.sent == [] + old(tx, pending_for=7 * 3600) + reconcile.reconcile_once(net) + assert net.sent # absent and complete: the newest version is sent again + + +# --- abandonment only with proof --------------------------------------------------------------------------------- + + +def test_missing_inputs_with_no_server_knowing_it_abandons_and_frees_the_coins(world): + user, round_, tx = building(world) + net = Net(verdict=Verdict(Outcome.REFUSED, MISSING)) + assert reconcile.reconcile_once(net)["abandoned"] == 1 + tx.refresh_from_db() + assert tx.status == "failed" and MISSING in tx.failure_reason and tx.failed_at + assert Participation.objects.get().status == "rejected" and Movement.objects.get(kind="bet").status == "failed" + assert reservation.spendable(user).count() == 1 and balance_of(user).spendable_sats == 3 * BET + entry = AuditEntry.objects.get(action="tx_abandoned") + assert entry.payload["coins_freed"] == 1 + + +@pytest.mark.parametrize("verdict", [ + Verdict(Outcome.REFUSED, "insufficient fee, rejecting replacement"), # a refusal for the fee is not a proof + Verdict(Outcome.UNCERTAIN, "timeout"), +]) +def test_anything_but_missing_inputs_abandons_nothing(world, verdict): + user, round_, tx = building(world) + reconcile.reconcile_once(Net(verdict=verdict)) + tx.refresh_from_db() + assert tx.status in ("building", "pending") and tx.reserved_utxos.count() == 1 and not reservation.spendable(user).exists() + + +def test_an_accepted_resend_means_it_is_alive(world): + user, round_, tx = building(world) + assert reconcile.reconcile_once(Net(verdict=Verdict(Outcome.ACCEPTED, "")))["recovered"] == 1 + + +def test_a_network_that_did_not_answer_enough_proves_nothing(world): + user, round_, tx = building(world) + net = Net(complete=False, verdict=Verdict(Outcome.REFUSED, MISSING)) + assert reconcile.reconcile_once(net) == {"recovered": 0, "abandoned": 0, "resurrected": 0} and net.sent == [] + assert ChainTransaction.objects.get().status == "building" + + +@pytest.mark.parametrize("reason,expected", [ + ("bad-txns-inputs-missingorspent", True), ("missing-inputs", True), ("the transaction was rejected\n\nmissing inputs", True), + ("insufficient fee", False), ("mempool min fee not met", False), ("", False), +]) +def test_only_missing_or_spent_inputs_count_as_proof(reason, expected): + assert reconcile.is_missing_inputs(reason) is expected + + +def test_a_row_that_changed_meanwhile_is_not_overwritten(world): + user, round_, tx = building(world) + ChainTransaction.objects.filter(pk=tx.pk).update(status="pending") # promoted by someone else + assert reconcile._promote(tx.pk, timezone.now(), tx.current_txid) is False + + +# --- failed rows stay watched (TX-23) ----------------------------------------------------------------------------- + + +def failed(world): + user, round_, tx = building(world) + reconcile.reconcile_once(Net(verdict=Verdict(Outcome.REFUSED, MISSING))) + tx.refresh_from_db() + return tx + + +def test_a_failed_row_that_confirms_is_reported_once_and_pauses_new_bets(world): + tx = failed(world) + net = Net(confirmed={tx.pk: (tx.current_txid, 200)}) + assert reconcile.reconcile_once(net)["resurrected"] == 1 + assert reconcile.reconcile_once(net)["resurrected"] == 0 # once + assert AuditEntry.objects.filter(action="tx_resurrected").count() == 1 + assert LotteryConfig.get().new_bets_suspended + assert ChainTransaction.objects.get().status == "failed" # the reconciler does not fix the linked rows itself + + +def test_a_failed_row_is_watched_for_fourteen_days_only(world): + tx = failed(world) + ChainTransaction.objects.filter(pk=tx.pk).update(failed_at=timezone.now() - timedelta(days=15)) + assert reconcile.reconcile_once(Net(confirmed={tx.pk: (tx.current_txid, 200)}))["resurrected"] == 0 diff --git a/tests/django/test_deposits.py b/tests/django/test_deposits.py index 00c9e66..2d5751d 100644 --- a/tests/django/test_deposits.py +++ b/tests/django/test_deposits.py @@ -222,7 +222,7 @@ def test_the_operator_changes_the_threshold_from_the_admin_and_it_is_audited(cli client.force_login(admin) config = LotteryConfig.get() data = {f: getattr(config, f) for f in ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds", - "draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb")} + "draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb", "rbf_timeout_seconds")} data["deposit_confirmations"] = 3 assert client.post("/admin/lottery/lotteryconfig/1/change/", data).status_code == 302 assert LotteryConfig.get().deposit_confirmations == 3 diff --git a/tests/integration/test_confirmation_regtest.py b/tests/integration/test_confirmation_regtest.py new file mode 100644 index 0000000..7f59337 --- /dev/null +++ b/tests/integration/test_confirmation_regtest.py @@ -0,0 +1,84 @@ +"""B7 — confirmation, replace-by-fee and the adoption of an earlier version, on the local regtest +(TX-30..TX-32, TX-12, TX-19).""" + +import time +from datetime import timedelta + +import pytest +from django.utils import timezone + +from apps.lottery import bets, confirmation, rbf +from apps.lottery.ledger import ChainLedger +from apps.lottery.models import ChainTransaction, Movement, Participation, Utxo +from plm_engine.regtest import RegtestError +from tests.django.test_chain_bets import ScriptedChain +from tests.integration.test_bet_regtest import BET, PLM, funded_player, network, open_round # noqa: F401 +from uuid import uuid4 + +pytestmark = pytest.mark.django_db + + +def placed(user, network): + ChainLedger().place_bet(user, open_round().pk, request_id=uuid4()) + assert bets.process_bets(network)["accepted"] == 1 + return ChainTransaction.objects.get() + + +def poll_until(network, tx, status="confirmed"): + for _ in range(80): + confirmation.poll_confirmations(network) + tx.refresh_from_db() + if tx.status == status: + return + time.sleep(0.25) + + +def age(tx): + ChainTransaction.objects.filter(pk=tx.pk).update(last_broadcast_at=timezone.now() - timedelta(hours=1)) + + +def test_a_bet_is_confirmed_by_a_block_and_enters_the_draw(regtest_node, network, funded_player): + user, _ = funded_player + tx = placed(user, network) + regtest_node.mine(1) + poll_until(network, tx) + assert tx.status == "confirmed" + bet = Participation.objects.get() + assert bet.status == "confirmed" and bet.confirmed_at and bet.bet_txid == tx.current_txid + assert Movement.objects.get(kind="bet").status == "confirmed" + assert Utxo.objects.get(reserved_by=tx).spent_txid == tx.current_txid + + +def test_a_slow_bet_is_replaced_by_fee_and_the_replacement_confirms(regtest_node, network, funded_player): + user, _ = funded_player + tx = placed(user, network) + first = tx.current_txid + age(tx) + assert rbf.bump_due(network) == 1 + tx.refresh_from_db() + assert tx.current_txid != first and tx.fee_rate_sat_vb == 2 + assert regtest_node.rpc("getmempoolentry", tx.current_txid)["vsize"] > 0 # the node took the replacement... + with pytest.raises(RegtestError): + regtest_node.rpc("getmempoolentry", first) # ...and dropped the original + regtest_node.mine(1) + poll_until(network, tx) + assert tx.status == "confirmed" and Participation.objects.get().bet_txid == tx.current_txid + assert tx.versions.count() == 2 + + +def test_an_earlier_version_that_confirms_is_adopted(regtest_node, network, funded_player): + """The replacement never reached the node (the send was uncertain); the original is what the chain mines.""" + user, _ = funded_player + tx = placed(user, network) + original = tx.current_txid + age(tx) + from apps.lottery.electrum_chain import Outcome, Verdict + + assert rbf.bump(tx.pk, ScriptedChain(Verdict(Outcome.UNCERTAIN, "timeout"))) is not None # the row follows version 2 + tx.refresh_from_db() + assert tx.current_txid != original + regtest_node.mine(1) # version 1 is in the node's mempool and gets mined + poll_until(network, tx) + assert tx.status == "confirmed" and tx.current_txid == original # adopted (TX-31) + assert Participation.objects.get().bet_txid == original + assert Utxo.objects.get(reserved_by=tx).spent_txid == original