B7: confirmations with merkle proof, adoption of the confirmed version, RBF bumper and reconciler (TX-30..33, DAT-85)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
davideandClaude Sonnet 5.5 committed 2026-11-16 16:29:08 +01:00
1 parent feff993584
commit fa57d2e5ab
20 files changed
+1165 -20

No files matched your search

+1 -1
View File
@@ -8,7 +8,7 @@ The user communicates in Italian in chat — reply to them in Italian. Everythin
## Project status
The migration from FastAPI to Django is in progress on branch `django-migration` (plan: [roadmap.md](roadmap.md), steps A1–A10, B1–B6 done). The suite has 1288 tests: the old stack's under `tests/unit/`, the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Open defects of the old stack are in [BUGS.md](BUGS.md); accepted limits in [docs/limiti-noti.md](docs/limiti-noti.md); every closed `B-nn` is in [docs/storico-bug.md](docs/storico-bug.md) (`git log --all --grep 'B-nn'` finds its commit).
The migration from FastAPI to Django is in progress on branch `django-migration` (plan: [roadmap.md](roadmap.md), steps A1–A10, B1–B7 done). The suite has 1336 tests: the old stack's under `tests/unit/`, the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Open defects of the old stack are in [BUGS.md](BUGS.md); accepted limits in [docs/limiti-noti.md](docs/limiti-noti.md); every closed `B-nn` is in [docs/storico-bug.md](docs/storico-bug.md) (`git log --all --grep 'B-nn'` finds its commit).
**[docs/](docs/README.md) is the specification, and it is authoritative.** Every verifiable rule there has a stable requirement ID (`RND-04`, `BET-02`, …) with `*Codice:*`/`*Test:*` pointers to what implements it; the index is [docs/come-funziona.md](docs/come-funziona.md) and the ID convention and change workflow are in [docs/processo-di-modifica.md](docs/processo-di-modifica.md). A behaviour change starts in the spec: edit or add the requirement, implement it, cite the ID in the test docstring and the commit message, and ship spec + code + test in one commit. If this file and the spec disagree, the spec wins and this file is the one to fix. Before writing code, read the relevant spec documents and the "Architecture" section below.
+1 -1
View File
@@ -62,7 +62,7 @@ python -m pytest # all tests
python -m pytest tests/unit/test_hd.py # one file
```
1288 unit tests cover HD derivation, PSBT building, the Electrum client, bets,
1336 unit tests cover HD derivation, PSBT building, the Electrum client, bets,
deposits, withdrawals, the round/draw engine, RBF fee-bumping, admin config,
the pending-inclusive balance calculation, and the SSE push channel. No
automated integration tests against a live Electrum connection — mainnet
+1 -1
View File
@@ -83,7 +83,7 @@ class LotteryConfigAdmin(admin.ModelAdmin):
commands, which ask for a reason (WEB-43). A change is audited with what changed."""
fields = ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds",
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb")
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb", "rbf_timeout_seconds")
def has_add_permission(self, request):
return False
+10 -8
View File
@@ -77,6 +77,8 @@ def _sign_one(tx: ChainTransaction, account, fee_rate: int) -> bool:
if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid is not None:
return False # someone else got there first
register_version(fresh, built.txid, built.raw_hex)
fresh.fee_rate_sat_vb = fee_rate # DAT-85: what the next bump starts from
fresh.save(update_fields=["fee_rate_sat_vb"])
bet.bet_txid, bet.bet_vout, bet.bet_amount_sats = built.txid, RECIPIENT_VOUT, built.recipient_sats
bet.save(update_fields=["bet_txid", "bet_vout", "bet_amount_sats"])
audit.record("bet_signed", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk,
@@ -85,23 +87,23 @@ def _sign_one(tx: ChainTransaction, account, fee_rate: int) -> bool:
return True
def _fail(tx: ChainTransaction, reason: str, *, release: bool) -> None:
def _fail(tx: ChainTransaction, reason: str, *, release: bool, action: str = "bet_refused") -> int:
"""Refused with proof (or unbuildable): the bet is rejected, the coins return, the movement stops counting."""
with transaction.atomic():
fresh = ChainTransaction.objects.select_for_update().get(pk=tx.pk)
if fresh.status not in (ChainTransaction.Status.BUILDING, ChainTransaction.Status.PENDING):
return
return 0
fresh.status, fresh.failed_at, fresh.failure_reason = ChainTransaction.Status.FAILED, timezone.now(), reason[:500]
fresh.save(update_fields=["status", "failed_at", "failure_reason"])
if release:
reservation.release(fresh)
freed = reservation.release(fresh) if release else 0
bet = _bet_of(fresh)
if bet is not None:
bet.status = Participation.Status.REJECTED
bet.save(update_fields=["status"])
Movement.objects.filter(participation=bet, kind=Movement.Kind.BET).update(status=Movement.Status.FAILED)
audit.record("bet_refused", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk,
outcome="error", payload={"reason": reason[:500]})
audit.record(action, actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk,
outcome="error", payload={"reason": reason[:500], "coins_freed": freed})
return freed
def _settle(tx_id: int, txid: str, verdict, now: datetime) -> bool:
@@ -111,8 +113,8 @@ def _settle(tx_id: int, txid: str, verdict, now: datetime) -> bool:
if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid != txid:
return False
if verdict.outcome is Outcome.ACCEPTED:
fresh.status, fresh.broadcast_at = ChainTransaction.Status.PENDING, now
fresh.save(update_fields=["status", "broadcast_at"])
fresh.status, fresh.broadcast_at, fresh.last_broadcast_at = ChainTransaction.Status.PENDING, now, now
fresh.save(update_fields=["status", "broadcast_at", "last_broadcast_at"])
audit.record("bet_broadcast", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk,
payload={"txid": txid, "note": verdict.reason})
return True
+88
View File
@@ -0,0 +1,88 @@
"""Confirmations of the platform's own transactions (roadmap B7, TX-30, TX-31, TX-26).
Three phases like the deposit scan: read the pending transactions and every txid each has had; ask the servers with no
database open (a version in a block, with a merkle proof the others back); write the outcome, only for rows still
`pending`. The version that confirms is adopted even when it is not `current_txid`: an RBF replacement may not have won.
"""
import asyncio
import logging
from datetime import datetime
from django.db import transaction
from django.utils import timezone
from . import audit
from .models import ChainTransaction, DepositAddress, Movement, Participation, Utxo
logger = logging.getLogger(__name__)
WORKER = "worker"
def spending_address(tx: ChainTransaction) -> str | None:
"""The address whose history shows the transaction: the player's for a bet or a withdrawal, the round's pool for a
payout (B8)."""
if tx.kind == ChainTransaction.Kind.PAYOUT:
return tx.round.pool_address if tx.round_id else None
owner = DepositAddress.objects.filter(user_id=tx.user_id).first()
return owner.address if owner else None
def plan() -> list[tuple[int, str, frozenset[str]]]:
wanted = []
for tx in ChainTransaction.objects.filter(status=ChainTransaction.Status.PENDING).select_related("round").order_by("id"):
address = spending_address(tx)
txids = frozenset(tx.versions.values_list("txid", flat=True))
if address and txids:
wanted.append((tx.pk, address, txids))
return wanted
def apply_confirmations(confirmed: dict[int, tuple[str, int]], now: datetime | None = None) -> int:
now = now or timezone.now()
done = 0
for tx_id, (txid, height) in confirmed.items():
with transaction.atomic():
fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id)
if fresh.status != ChainTransaction.Status.PENDING:
continue # TX-26: confirmed, abandoned or settled by someone else while the network was asked
version = fresh.versions.filter(txid=txid).first()
if version is None:
logger.error("transaction %s: confirmed txid %s is none of its versions", tx_id, txid)
continue
adopted = fresh.current_txid != txid
if adopted: # TX-31
fresh.current_txid = txid
if version.raw_hex:
fresh.raw_hex = version.raw_hex
fresh.status, fresh.confirmed_at = ChainTransaction.Status.CONFIRMED, now
fresh.save(update_fields=["status", "confirmed_at", "current_txid", "raw_hex"])
Utxo.objects.filter(reserved_by=fresh).update(spent_txid=txid)
if fresh.kind == ChainTransaction.Kind.BET:
_bet_confirmed(fresh, txid, now)
audit.record("tx_confirmed", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk,
payload={"txid": txid, "height": height, "adopted": adopted, "kind": fresh.kind})
done += 1
return done
def _bet_confirmed(tx: ChainTransaction, txid: str, now: datetime) -> None:
"""The bet takes part in the draw (BET, TX-07): found through its fixed links, never through the txid (TX-08)."""
movement = Movement.objects.filter(chain_transaction=tx, kind=Movement.Kind.BET).select_related("participation").first()
if movement is None or movement.participation is None:
return
bet = movement.participation
if bet.status == Participation.Status.PENDING:
bet.status, bet.confirmed_at, bet.bet_txid = Participation.Status.CONFIRMED, now, txid
bet.save(update_fields=["status", "confirmed_at", "bet_txid"])
if movement.status == Movement.Status.PENDING:
movement.status, movement.confirmed_at = Movement.Status.CONFIRMED, now
movement.save(update_fields=["status", "confirmed_at"])
def poll_confirmations(chain, now: datetime | None = None) -> int:
wanted = plan()
if not wanted:
return 0
confirmed = asyncio.run(chain.observe_confirmations(wanted))
return apply_confirmations(confirmed, now) if confirmed else 0
+71 -2
View File
@@ -176,15 +176,20 @@ class ElectrumChain:
if not self.majority(await self._ask_others(lists_it)):
return False
return await self._proof_holds(active, entry.txid, entry.height)
async def _proof_holds(self, active: ElectrumClient, txid: str, height: int) -> bool:
"""ELC-26, TX-22: a header of the active server at `height` with valid work for the profile, and a merkle
proof from the others (by majority) that puts `txid` in that block."""
try:
header = await asyncio.wait_for(active.get_block_header(entry.height), ANSWER_TIMEOUT)
header = await asyncio.wait_for(active.get_block_header(height), ANSWER_TIMEOUT)
except Exception:
return False
if not header_meets_its_own_target(str(header), self.profile.pow_limit):
return False
async def proves_it(client: ElectrumClient) -> bool:
return proof_matches_header(entry.txid, entry.height, await client.get_merkle(entry.txid, entry.height), str(header))
return proof_matches_header(txid, height, await client.get_merkle(txid, height), str(header))
return self.majority(await self._ask_others(proves_it))
@@ -197,6 +202,70 @@ class ElectrumChain:
return self.majority(await self._ask_others(lacks_it))
# -- confirmations and presence (TX-30, TX-33) ----------------------------------------------------------------
@staticmethod
async def _history(client: ElectrumClient, address: str) -> list[tuple[str, int]] | None:
try:
raw = await asyncio.wait_for(client.get_history(address_to_scripthash(address)), ANSWER_TIMEOUT)
except Exception as error:
logger.warning("history of %s failed: %s", address, error)
return None
try:
return [(str(item["tx_hash"]), int(item["height"])) for item in raw]
except (KeyError, TypeError, ValueError):
return None
async def observe_confirmations(self, wanted: list[tuple[int, str, frozenset[str]]]) -> dict[int, tuple[str, int]]:
"""`wanted` is `(row id, spending address, every txid of the row)`. Returns, per row, the version that is in a
block *and* proven (TX-30): `{row id: (txid, height)}`. An address that cannot be read blocks nobody else."""
active = await self._connect_active()
if active is None:
return {}
try:
histories: dict[str, list[tuple[str, int]] | None] = {}
confirmed: dict[int, tuple[str, int]] = {}
for row_id, address, txids in wanted:
if address not in histories:
histories[address] = await self._history(active, address)
history = histories[address]
if history is None:
continue
for txid, height in history:
if txid in txids and height > 0 and await self._proof_holds(active, txid, height):
confirmed[row_id] = (txid, height)
break
return confirmed
except Exception:
logger.exception("confirmation poll failed")
return {}
finally:
await active.close()
async def versions_known(self, address: str, txids: frozenset[str]) -> tuple[set[str], bool]:
"""TX-33: which of `txids` any server of the network has seen (mempool or block), and whether the network
answered enough for the *absence* of the rest to mean something (the active server and the quorum)."""
active = await self._connect_active()
if active is None:
return set(), False
try:
mine = await self._history(active, address)
async def theirs(client: ElectrumClient):
history = await self._history(client, address)
return None if history is None else {txid for txid, _ in history if txid in txids}
others = await self._ask_others(theirs)
known = ({txid for txid, _ in mine if txid in txids} if mine is not None else set())
for answer in others:
if answer:
known |= answer
# an absence is a statement only if the active server answered and the others agree among themselves
complete = mine is not None and self.majority([a is not None for a in others])
return known, complete
finally:
await active.close()
# -- broadcast (BET-32) -----------------------------------------------------------------------------------
async def transaction_known(self, active: ElectrumClient, txid: str) -> bool | None:
@@ -42,13 +42,18 @@ def process_chain_bets() -> None:
"""BET-31/32: sign what the web admitted, send what was signed. On a real ledger only."""
from django.conf import settings
from apps.lottery import bets
from apps.lottery import bets, confirmation, rbf, reconcile
from apps.lottery.electrum_chain import ElectrumChain
chain = ElectrumChain(settings.ELECTRUM_ENDPOINTS, settings.PLM_PROFILE, settings.ELECTRUM_ALLOW_NO_QUORUM)
done = bets.process_bets(chain)
if any(done.values()):
logger.info("bets: %s", done)
confirmed = confirmation.poll_confirmations(chain) # B7: TX-30
bumped = rbf.bump_due(chain) # TX-32
reconciled = reconcile.reconcile_once(chain) # TX-33
if confirmed or bumped or any(reconciled.values()):
logger.info("transactions: confirmed=%s bumped=%s reconciled=%s", confirmed, bumped, reconciled)
def tick() -> None:
@@ -0,0 +1,37 @@
# Generated by Django 5.2.19 on 2026-11-16 15:20
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('lottery', '0007_fee_rate'),
]
operations = [
migrations.AddField(
model_name='chaintransaction',
name='attempt_count',
field=models.PositiveSmallIntegerField(default=1),
),
migrations.AddField(
model_name='chaintransaction',
name='fee_rate_sat_vb',
field=models.PositiveIntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='chaintransaction',
name='last_broadcast_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='lotteryconfig',
name='rbf_timeout_seconds',
field=models.PositiveIntegerField(default=900),
),
migrations.AddConstraint(
model_name='lotteryconfig',
constraint=models.CheckConstraint(condition=models.Q(('rbf_timeout_seconds__gte', 60), ('rbf_timeout_seconds__lte', 86400)), name='lottery_config_rbf_timeout_range'),
),
]
+9
View File
@@ -37,6 +37,8 @@ class LotteryConfig(models.Model):
max_round_pool_sats = models.BigIntegerField(default=0) # 0 = no cap
# Network fee of the transactions built from now on, sat/vB (BET-04); a transaction in flight keeps its own.
fee_rate_sat_vb = models.PositiveIntegerField(default=1)
# TX-32: how long a pending transaction waits before it is replaced with a higher fee.
rbf_timeout_seconds = models.PositiveIntegerField(default=900)
# ADR-57: three distinct commands.
pause_after_round = models.BooleanField(default=False)
@@ -72,6 +74,9 @@ class LotteryConfig(models.Model):
models.CheckConstraint(
condition=Q(fee_rate_sat_vb__gte=1, fee_rate_sat_vb__lte=5000), name="lottery_config_fee_rate_range"
),
models.CheckConstraint(
condition=Q(rbf_timeout_seconds__gte=60, rbf_timeout_seconds__lte=86400), name="lottery_config_rbf_timeout_range"
),
]
@classmethod
@@ -245,6 +250,10 @@ class ChainTransaction(models.Model):
confirmed_at = models.DateTimeField(null=True, blank=True)
failed_at = models.DateTimeField(null=True, blank=True)
failure_reason = models.TextField(blank=True, default="")
# DAT-85: the current version's rate, the last send (RBF moves it; broadcast_at stays the first) and the tries.
fee_rate_sat_vb = models.PositiveIntegerField(null=True, blank=True)
last_broadcast_at = models.DateTimeField(null=True, blank=True)
attempt_count = models.PositiveSmallIntegerField(default=1)
class Meta:
indexes = [models.Index(fields=["status"])]
+160
View File
@@ -0,0 +1,160 @@
"""Replace-by-fee for the platform's transactions (roadmap B7, TX-32, TX-10..TX-13, TX-24).
Same coins, same outputs to the counterpart; the increase comes out of one output only, the player's change. The
replacement is signed and **recorded as a version before it is sent**; after the send (accepted or uncertain) the row
moves to it with an update that matches only if the row is still `pending` and still on the txid that was replaced.
"""
import asyncio
import logging
from datetime import datetime, timedelta
from django.db import transaction
from django.utils import timezone
from embit import script
from embit.finalizer import finalize_psbt
from embit.psbt import PSBT
from embit.transaction import Transaction, TransactionInput, TransactionOutput
from plm_engine import hd
from plm_engine.psbt_builder import DUST_LIMIT_SATS, MAX_FEE_RATE_SAT_VB, RBF_SEQUENCE, estimate_vsize
from . import audit, keys
from .chain import register_version
from .electrum_chain import Outcome
from .models import AuditEntry, ChainTransaction, DepositAddress, LotteryConfig, Participation
logger = logging.getLogger(__name__)
WORKER = "worker"
FEE_RATE_INCREMENT = 1 # sat/vB more than the recorded rate, per bump (TX-10)
INCREMENTAL_RELAY_FEE_RATE = 1 # BIP125 rule 4, sat/vB
REASON_MAX_RATE = "max_fee_rate"
REASON_NO_CHANGE = "no_change_output"
REASON_REFUSED = "replacement_refused"
class RbfError(Exception):
def __init__(self, message: str, reason: str):
super().__init__(message)
self.reason = reason
def due(now: datetime | None = None) -> list[ChainTransaction]:
"""TX-32: pending bets whose last send is older than the timeout."""
now = now or timezone.now()
limit = now - timedelta(seconds=LotteryConfig.get().rbf_timeout_seconds)
rows = ChainTransaction.objects.filter(kind=ChainTransaction.Kind.BET, status=ChainTransaction.Status.PENDING)
return [t for t in rows.order_by("id") if (t.last_broadcast_at or t.broadcast_at or t.created_at) <= limit]
def _build_replacement(tx: ChainTransaction, account) -> tuple[str, str, int, int]:
"""Phase 2, no network and no open transaction: the signed replacement. Returns `(txid, raw hex, new rate, vsize)`."""
if (tx.fee_rate_sat_vb or 1) >= MAX_FEE_RATE_SAT_VB:
raise RbfError(f"already at the maximum fee rate ({MAX_FEE_RATE_SAT_VB} sat/vB)", REASON_MAX_RATE)
owner = DepositAddress.objects.get(user=tx.user)
key = hd.user_key(account, owner.derivation_index)
own_script = script.p2wpkh(key.to_public())
old = Transaction.parse(bytes.fromhex(tx.raw_hex))
amounts = {(u.txid, u.vout): u.amount_sats for u in tx.reserved_utxos.all()}
input_amounts = [amounts[(v.txid.hex(), v.vout)] for v in old.vin]
old_fee = sum(input_amounts) - sum(o.value for o in old.vout)
vsize = estimate_vsize(len(old.vin), len(old.vout))
rate = tx.fee_rate_sat_vb or 1
target_rate = min(rate + FEE_RATE_INCREMENT, MAX_FEE_RATE_SAT_VB)
min_delta = vsize * INCREMENTAL_RELAY_FEE_RATE # BIP125: never below this, whatever the target says
delta = max(vsize * target_rate - old_fee, min_delta)
change = next((i for i, o in enumerate(old.vout) if o.script_pubkey.data == own_script.data), None)
if change is None:
raise RbfError("there is no change output to take the increase from", REASON_NO_CHANGE)
available = old.vout[change].value
if available < min_delta:
raise RbfError(f"its change ({available} sat) cannot cover the minimum increase ({min_delta} sat)", REASON_NO_CHANGE)
outputs = list(old.vout)
if available - delta >= DUST_LIMIT_SATS:
outputs[change] = TransactionOutput(available - delta, outputs[change].script_pubkey)
else: # TX-11: nothing relayable would be left, so the output goes to the fee and this is the last replacement
del outputs[change]
delta = available
new = Transaction(vin=[TransactionInput(v.txid, v.vout, sequence=RBF_SEQUENCE) for v in old.vin], vout=outputs)
psbt = PSBT(new)
for i, amount in enumerate(input_amounts):
psbt.inputs[i].witness_utxo = TransactionOutput(amount, own_script)
if psbt.sign_with(key) != len(old.vin):
raise RuntimeError("the replacement was not signed on every input")
final = finalize_psbt(psbt)
if final is None:
raise RuntimeError("failed to finalize the replacement")
new_rate = (old_fee + delta) // estimate_vsize(len(old.vin), len(outputs))
return final.txid().hex(), final.serialize().hex(), new_rate, vsize
def _record_failure(tx: ChainTransaction, reason: str, message: str) -> None:
"""TX-24: once per (txid, reason); the log carries every repeat."""
seen = AuditEntry.objects.filter(action="rbf_bump_failed", subject_id=str(tx.pk)).values_list("payload", flat=True)
if any(p.get("txid") == tx.current_txid and p.get("reason") == reason for p in seen):
logger.info("bump of transaction %s failed again (%s): %s", tx.pk, reason, message)
return
audit.record("rbf_bump_failed", actor_label=WORKER, subject_type="chain_transaction", subject_id=tx.pk,
outcome=AuditEntry.Outcome.ERROR, reason=message,
payload={"txid": tx.current_txid, "reason": reason})
def bump(tx_id: int, chain, now: datetime | None = None) -> str | None:
"""Accelerate one transaction. Returns the new txid, or None if there was nothing to do (not pending any more, a
conflict, or an impossible/refused bump, which is recorded)."""
now = now or timezone.now()
tx = ChainTransaction.objects.select_related("user").filter(pk=tx_id).first()
if tx is None or tx.status != ChainTransaction.Status.PENDING:
return None
try:
account = keys.signing_account_key()
new_txid, raw_hex, new_rate, _ = _build_replacement(tx, account)
except RbfError as error:
_record_failure(tx, error.reason, str(error))
return None
except keys.SigningKeyUnavailable as error:
logger.error("cannot bump transaction %s: %s", tx_id, error)
return None
old_txid = tx.current_txid
with transaction.atomic(): # the candidate is recorded before it is sent (TX-12)
fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id)
if fresh.status != ChainTransaction.Status.PENDING or fresh.current_txid != old_txid:
return None
from .models import TransactionVersion
if not TransactionVersion.objects.filter(txid=new_txid).exists():
TransactionVersion.objects.create(transaction=fresh, txid=new_txid, raw_hex=raw_hex)
verdict = asyncio.run(chain.broadcast(raw_hex, new_txid))
if verdict.outcome is Outcome.REFUSED: # the candidate stays recorded; nobody will confirm it (TX-24)
_record_failure(tx, REASON_REFUSED, f"the network refused the replacement: {verdict.reason}")
return None
with transaction.atomic(): # TX-12: move the row, only if it is still exactly what was replaced
moved = ChainTransaction.objects.filter(
pk=tx_id, status=ChainTransaction.Status.PENDING, current_txid=old_txid
).update(current_txid=new_txid, raw_hex=raw_hex, fee_rate_sat_vb=new_rate, last_broadcast_at=now,
attempt_count=tx.attempt_count + 1)
if not moved: # TX-13: it changed under us; the candidate stays and confirmation looks for it anyway
logger.warning("transaction %s changed while its replacement was being sent: conflict", tx_id)
return None
Participation.objects.filter(movements__chain_transaction_id=tx_id, bet_txid=old_txid).update(bet_txid=new_txid)
audit.record("rbf_bumped", actor_label=WORKER, subject_type="chain_transaction", subject_id=tx_id,
payload={"from": old_txid, "to": new_txid, "fee_rate_sat_vb": new_rate,
"network": verdict.outcome.value})
return new_txid
def bump_due(chain, now: datetime | None = None) -> int:
"""TX-32: accelerate every transaction that waited too long. An error on one row stops no other."""
count = 0
for tx in due(now):
try:
if bump(tx.pk, chain, now):
count += 1
except Exception:
logger.exception("bumping transaction %s failed", tx.pk)
return count
+125
View File
@@ -0,0 +1,125 @@
"""Reconciling what the database believes with what the network knows (roadmap B7, TX-33, TX-15..TX-17, TX-23, TX-26).
A row is only ever given up on proof (ADR-51): a refusal of the newest version for missing or already spent inputs
while no server knows any version of it. A timeout, silence or a refusal for the fee is never a proof. The second
proof of TX-17 (an input spent in a block by another transaction) arrives with the reorg work of B10.
"""
import asyncio
import logging
from datetime import datetime, timedelta
from django.db import transaction
from django.utils import timezone
from . import audit
from .bets import WORKER, _fail
from .confirmation import apply_confirmations, spending_address
from .electrum_chain import Outcome
from .models import ChainTransaction, LotteryConfig
logger = logging.getLogger(__name__)
GRACE_BUILDING = timedelta(seconds=120)
GRACE_PENDING = timedelta(hours=6)
FAILED_WATCH = timedelta(days=14)
# Wordings by which a node says the inputs are not there to spend. Anything else is not a proof (TX-17).
_MISSING_INPUTS = ("missing-inputs", "missing inputs", "missingorspent", "already spent")
def is_missing_inputs(reason: str) -> bool:
lowered = reason.lower()
return any(marker in lowered for marker in _MISSING_INPUTS)
def _is_due(tx: ChainTransaction, now: datetime) -> bool:
if tx.status == ChainTransaction.Status.BUILDING:
return tx.current_txid is not None and now - tx.created_at >= GRACE_BUILDING
return now - (tx.broadcast_at or tx.created_at) >= GRACE_PENDING
def _promote(tx_id: int, now: datetime, txid: str) -> bool:
"""TX-16: the network has it. A `building` row becomes `pending` (so confirmation and RBF can take over)."""
with transaction.atomic():
fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id)
if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid != txid: # TX-26
return False
fresh.status, fresh.broadcast_at, fresh.last_broadcast_at = ChainTransaction.Status.PENDING, now, now
fresh.save(update_fields=["status", "broadcast_at", "last_broadcast_at"])
audit.record("pending_tx_recovered", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk,
payload={"txid": txid})
return True
def reconcile_once(chain, now: datetime | None = None) -> dict[str, int]:
now = now or timezone.now()
done = {"recovered": 0, "abandoned": 0, "resurrected": 0}
rows = ChainTransaction.objects.filter(
status__in=[ChainTransaction.Status.BUILDING, ChainTransaction.Status.PENDING]
).select_related("round").order_by("id")
for tx in rows:
if not _is_due(tx, now):
continue
try:
_reconcile_row(tx, chain, now, done)
except Exception:
logger.exception("reconciling transaction %s failed", tx.pk)
_watch_failed(chain, now, done)
return done
def _reconcile_row(tx: ChainTransaction, chain, now: datetime, done: dict[str, int]) -> None:
address = spending_address(tx)
txids = frozenset(tx.versions.values_list("txid", flat=True))
if not address or not txids:
return
known, complete = asyncio.run(chain.versions_known(address, txids))
if known: # TX-15: at least one server has at least one version: it is alive
if _promote(tx.pk, now, tx.current_txid):
done["recovered"] += 1
return
if not complete or not tx.raw_hex: # an unreadable network proves nothing
return
verdict = asyncio.run(chain.broadcast(tx.raw_hex, tx.current_txid))
if verdict.outcome is Outcome.ACCEPTED:
if _promote(tx.pk, now, tx.current_txid):
done["recovered"] += 1
elif verdict.outcome is Outcome.REFUSED:
if is_missing_inputs(verdict.reason): # TX-17, proof 2
_fail(tx, f"abandoned: {verdict.reason}", release=True, action="tx_abandoned")
done["abandoned"] += 1
elif _promote(tx.pk, now, tx.current_txid): # refused for the fee: not a proof; let RBF accelerate it
done["recovered"] += 1
def _watch_failed(chain, now: datetime, done: dict[str, int]) -> None:
"""TX-23: a transaction given up for dead can still confirm. If one does, say it once and stop new bets."""
rows = ChainTransaction.objects.filter(
status=ChainTransaction.Status.FAILED, failed_at__gte=now - FAILED_WATCH
).select_related("round")
wanted = []
for tx in rows:
if audit_exists("tx_resurrected", tx.pk):
continue
address, txids = spending_address(tx), frozenset(tx.versions.values_list("txid", flat=True))
if address and txids:
wanted.append((tx.pk, address, txids))
if not wanted:
return
for tx_id, (txid, height) in asyncio.run(chain.observe_confirmations(wanted)).items():
with transaction.atomic():
if audit_exists("tx_resurrected", tx_id):
continue
audit.record("tx_resurrected", actor_label=WORKER, subject_type="chain_transaction", subject_id=tx_id,
outcome="error", payload={"txid": txid, "height": height})
LotteryConfig.objects.filter(pk=1).update(new_bets_suspended=True) # the operator puts things in order
done["resurrected"] += 1
def audit_exists(action: str, subject_id: int) -> bool:
from .models import AuditEntry
return AuditEntry.objects.filter(action=action, subject_id=str(subject_id)).exists()
__all__ = ["apply_confirmations", "is_missing_inputs", "reconcile_once"]
+2 -2
View File
@@ -4,7 +4,7 @@
## Project status (old stack)
All 10 stages of the original build order are code-complete and unit-tested — 1288 tests, the old stack's under `tests/unit/` and the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Beyond them: Docker + Caddy deployment, admin dashboard (`/admin`), static test UI (`/`), pending-inclusive balance display, an SSE push channel layered over the original polling, self-service password change + admin password reset, and the reconciliation/corroboration machinery below.
All 10 stages of the original build order are code-complete and unit-tested — 1336 tests, the old stack's under `tests/unit/` and the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Beyond them: Docker + Caddy deployment, admin dashboard (`/admin`), static test UI (`/`), pending-inclusive balance display, an SSE push channel layered over the original polling, self-service password change + admin password reset, and the reconciliation/corroboration machinery below.
Verified on mainnet with real money: registration + address derivation, deposit crediting (at the time, 1-conf; deposits now need `deposit_confirmations`), a real 10 PLM bet (broadcast → confirmed → change credited back), and one full round cycle (close → draw on a real block hash → 70/30 payout with sat math checked against the broadcast tx → confirmation → close → next round auto-opened). **Withdrawal and the RBF bump path have never been exercised against a live broadcast** — unit-tested only.
@@ -30,7 +30,7 @@ PYTHONPATH=. python scripts/decrypt_master_key.py # ops recovery: decrypt+pr
PYTHONPATH=. python scripts/encrypt_master_key.py # ops bootstrap: import an externally-generated xprv (--overwrite to replace)
PYTHONPATH=. python scripts/electrum_smoke_test.py # manual check: connect, handshake, subscribe to headers, print the tip
python -m pytest # all 1288 tests
python -m pytest # all 1336 tests
python -m pytest tests/unit/test_hd.py # one file
python -m pytest tests/unit/test_hd.py::test_derivation_is_deterministic # one test
```
+5 -1
View File
@@ -1500,7 +1500,7 @@ Le tabelle di catena (UTXO, versioni delle transazioni, depositi, indici HD) son
si elimina. Ogni importo è un intero in unità minime ([DAT-01]) e ogni timestamp è in UTC con fuso ([DAT-02]).
*Stato:* implementato (A2) · *Codice:* `apps/lottery/models.py` · *Test:* `tests/django/test_models.py::test_rows_that_carry_money_cannot_be_deleted_with_their_owner`
- **[DAT-77]** **Parametri e pause: `LotteryConfig`.** Una sola riga (`id = 1`, vincolo) con quota (10 PLM),
durata (600 s), pausa fra round (30 s), conferme richieste a un deposito (1–100, default 6, [DAT-62]; si cambia da `/admin/` con audit e vale subito per gli accrediti successivi), conferme del blocco di estrazione (1–20, default 3), commissione di rete in sat/vB (1–5 000, default 1: vale per le transazioni costruite dopo), tetto del
durata (600 s), pausa fra round (30 s), conferme richieste a un deposito (1–100, default 6, [DAT-62]; si cambia da `/admin/` con audit e vale subito per gli accrediti successivi), tempo oltre il quale una transazione lenta si accelera (`rbf_timeout_seconds`, 60–86 400 s, default 900), conferme del blocco di estrazione (1–20, default 3), commissione di rete in sat/vB (1–5 000, default 1: vale per le transazioni costruite dopo), tetto del
montepremi (0 = nessuno) e i tre interruttori di [ADR-57](adr/ADR-57-due-pause-distinte-dopo-round-e-incidente.md):
`pause_after_round`, `new_bets_suspended`, `withdrawals_suspended`. Valori non positivi o fuori intervallo
sono rifiutati dal database. I parametri si **copiano su `Round` all'apertura** ([DAT-61]); la copia la fa il
@@ -1557,4 +1557,8 @@ Le tabelle di catena (UTXO, versioni delle transazioni, depositi, indici HD) son
Un output è un solo deposito e un `credited` ha sempre il movimento e la data di accredito.
*Stato:* tabella implementata (B4); il listener e l'accredito sono B5 · *Codice:* `apps/lottery/models.py:Deposit` ·
*Test:* `tests/django/test_chain_models.py::test_a_credited_deposit_has_its_movement_and_date`, `test_an_output_is_one_deposit`
- **[DAT-85]** **Stato di invio di una transazione.** `ChainTransaction` registra la commissione della versione corrente
(`fee_rate_sat_vb`), l'istante dell'ultimo invio (`last_broadcast_at`, che l'RBF aggiorna mentre `broadcast_at` resta il primo) e il numero di
tentativi (`attempt_count`). Sono ciò che il bumper e il riconciliatore leggono ([TX-32], [TX-33]).
*Stato:* implementato (B7) · *Codice:* `apps/lottery/models.py:ChainTransaction` · *Test:* `tests/django/test_chain_rbf.py`
+39
View File
@@ -420,3 +420,42 @@ Nel pannello `/admin` (vedi [guida-admin.md](guida-admin.md) e
correzioni in corso anche `rbf_bump_failed` ([TX-24]) e
`pending_tx_resurrected` ([TX-23]);
- **Round** — stato di ogni round, vincitore e txid del pagamento.
## Transazioni sul sito Django (B7)
Il sito Django riscrive conferma, accelerazione e riconciliazione sull'ORM (`ChainTransaction`, [DAT-83]). Per ora riguardano le
**scommesse**; prelievi e pagamenti (B8, B9) usano gli stessi meccanismi. Restano fuori, e sono B10: il rilevamento delle
riorganizzazioni (finestra degli ultimi header, [ELC-22]…[ELC-24]) e le monete già accreditate che poi spariscono ([DEP-06]…[DEP-08]).
- **[TX-30]** **Conferma verificata, con tutte le versioni.** Il worker legge la cronologia dell'indirizzo che spende ogni transazione
`pending` e cerca **ogni** versione ([DAT-83]); una versione con altezza maggiore di 0 è candidata e si confermerà solo con la prova
merkle di [TX-22] (header dell'altezza con lavoro valido per il profilo, prova data dagli altri server a maggioranza,
[ELC-14]). Senza prova la riga resta `pending` e si riprova. Tre fasi: lettura del database, rete senza database aperto, scrittura.
*Stato:* implementato (B7) · *Codice:* `apps/lottery/confirmation.py`, `apps/lottery/electrum_chain.py:ElectrumChain.observe_confirmations` ·
*Test:* `tests/django/test_chain_confirmation.py`, `tests/integration/test_confirmation_regtest.py`
- **[TX-31]** **Si adotta la versione confermata.** Se a confermarsi è una versione **diversa** da `current_txid` (l'originale mentre
una sostituzione era in volo, o una sostituzione che il processo non aveva fatto in tempo a registrare), la riga adotta quel txid e quella
transazione grezza, e lo propaga a scommessa (`bet_txid`) e monete spese (`spent_txid`). Alla conferma di una scommessa: la
`Participation` diventa `confirmed` con `confirmed_at` ed entra nell'estrazione, il movimento di puntata diventa `confirmed`, le monete
spese risultano spese, e l'audit scrive `tx_confirmed`; tutto nella stessa transazione del database ([TX-26]).
*Stato:* implementato (B7) · *Codice:* `apps/lottery/confirmation.py:apply_confirmations` ·
*Test:* `tests/django/test_chain_confirmation.py::test_a_bet_is_confirmed_and_enters_the_draw`, `test_an_earlier_version_that_confirms_is_adopted`
- **[TX-32]** **Accelerazione (RBF).** Una transazione `pending` il cui ultimo invio è più vecchio di `rbf_timeout_seconds`
(`LotteryConfig`, 900 s) è ricostruita con una commissione di 1 sat/vB in più (mai oltre 5 000, mai meno dell'aumento minimo di BIP125),
spendendo le stesse monete; l'aumento si toglie **solo** dal resto del giocatore, e se ne resterebbe meno di 294 sat il resto sparisce
e quella è l'ultima accelerazione ([TX-10], [TX-11]). La sostituzione firmata è registrata come versione **prima** di trasmetterla
([TX-12]); dopo l'invio (accettato o incerto) la riga passa alla nuova versione con un aggiornamento condizionato a stato `pending` e
`current_txid` invariato, altrimenti è un conflitto e la candidata resta registrata ([TX-13]). Se la rete rifiuta la sostituzione o
l'accelerazione è impossibile, la riga non cambia e si scrive `rbf_bump_failed` una sola volta per coppia (txid, motivo) ([TX-24]).
*Stato:* implementato (B7) · *Codice:* `apps/lottery/rbf.py` ·
*Test:* `tests/django/test_chain_rbf.py`, `tests/integration/test_confirmation_regtest.py`
- **[TX-33]** **Riconciliazione e abbandono solo con prova.** Il worker esamina le transazioni `building` da oltre 120 s e `pending` da oltre
6 ore dal primo invio. Se almeno un server conosce una versione, la riga è viva: `building` passa a `pending` (`pending_tx_recovered`).
Se nessuno la conosce, si **ritrasmette** la versione più recente: accettata = viva; rifiutata con prova **per ingressi mancanti o già spesi**
= abbandono (la transazione è `failed` con il motivo, le monete ancora impegnate da *questa* transazione tornano libere, la scommessa è
respinta, il movimento non conta più, `tx_abandoned`); rifiutata per altro (commissione) o incerta = nessun abbandono. Le righe `failed`
restano sorvegliate 14 giorni: se una loro versione risulta confermata si scrive una sola volta `tx_resurrected` e le nuove puntate
si sospendono per l'operatore ([TX-23]). Non esiste un tempo massimo che abbandoni una transazione che un server tiene in mempool.
*Stato:* implementato (B7); la seconda prova di [TX-17] (ingresso speso in un blocco da un'altra transazione) è B10 ·
*Codice:* `apps/lottery/reconcile.py` · *Test:* `tests/django/test_chain_reconcile.py`
+2 -2
View File
@@ -45,10 +45,10 @@ Uscita: flusso completo su regtest; un riavvio durante il broadcast, due richies
- [x] **B4 · Modelli di catena.** UTXO, transazioni con tutte le versioni TXID, depositi, assegnazione dell'indice HD; la chiave di firma è accessibile solo al processo worker. *Requisiti:* DAT, TX. *Test:* vincoli, derivazione identica a parità di input.
- [x] **B5 · Listener e depositi reali.** Listener Electrum sull'ORM con rotazione, header validati, quorum e soglie di conferma al posto del deposito simulato. *Requisiti:* DEP, ELC. *Test:* unit con server finto + integrazione su regtest.
- [x] **B6 · Prenotazione UTXO e puntata reale.** Prenotazione atomica degli UTXO nel database, identificativo idempotente, «salva operazione e transazione firmata prima del broadcast», verdetto del broadcast (accettato, rifiutato, incerto). *Requisiti:* BET, TX-26. *Test:* due richieste concorrenti non spendono gli stessi input; timeout ambiguo riconciliato.
- [ ] **B7 · Conferme, riconciliazione e RBF.** Poller, riconciliatore e bumper sull'ORM con tutte le versioni TXID e adozione della versione confermata. *Requisiti:* TX. *Test:* conferma di una versione RBF precedente, ripresa dopo riavvio.
- [x] **B7 · Conferme, riconciliazione e RBF.** Poller, riconciliatore e bumper sull'ORM con tutte le versioni TXID e adozione della versione confermata. *Requisiti:* TX. *Test:* conferma di una versione RBF precedente, ripresa dopo riavvio.
- [ ] **B8 · Estrazione e pagamento reali.** Commit-reveal con blocco a quorum, verifica del montepremi, pagamento 70/30 con riserva per l'RBF, pausa automatica sui casi bloccanti. *Requisiti:* RND. *Test:* round completo su regtest, riorganizzazione simulata.
- [ ] **B9 · Prelievi e storico.** Prelievo con indirizzo, importo, commissione e netto previsti; storico con stato dell'operazione e TXID corrente con link all'explorer. *Requisiti:* WDR, WEB-24. *Test:* prelievo su regtest, sospensione dei prelievi.
- [ ] **B10 · Robustezza e chiusura di fase.** Test di riavvio durante il broadcast, richieste concorrenti web/worker, riorganizzazione; sostituzione definitiva dei blocchi in memoria. *Requisiti:* ARC-70…72, ADR-55.
- [ ] **B10 · Robustezza e chiusura di fase.** Test di riavvio durante il broadcast, richieste concorrenti web/worker, riorganizzazione; sostituzione definitiva dei blocchi in memoria. **Eredità di B5 e B7:** finestra degli ultimi header con rilevamento delle riorganizzazioni e pausa su quelle profonde (ELC-22…24), monete già accreditate che spariscono, spese altrove o reintegrate (DEP-06…08) e la seconda prova di abbandono di TX-17 (ingresso speso in un blocco da un'altra transazione). *Requisiti:* ARC-70…72, ADR-55.
- [ ] **B11 · Prova guidata del round su regtest.** Script `scripts/` che, su un database nuovo, crea giocatori con fondi, li fa puntare, fa scadere il round e **mina i blocchi su regtest** (B3) invece di aspettare il tempo; controlla chiusura, estrazione, pagamento e nuovo round, la conservazione dei fondi (vincita 70% più commissione 30% uguale al montepremi), l'ordine dell'audit e che la pagina pubblica `/risultati/<id>/` ricalcoli il vincitore senza nomi utente. Casi: quattro giocatori, un solo giocatore, puntata rifiutata, riavvio del worker durante l'estrazione. *Requisiti:* RND, WEB-12, WEB-13. *Test:* lo script esce con errore se un controllo fallisce; la prova fatta a mano con la catena simulata (4 giocatori, 40 PLM, 28 al vincitore, 12 alla piattaforma) è il riferimento.
## Fase C — Verifiche prima dell'uso pubblico
+176
View File
@@ -0,0 +1,176 @@
"""B7 — confirmations: TX-30, TX-31, TX-26, TX-08."""
import struct
from datetime import timedelta
import pytest
from django.utils import timezone
from embit.transaction import Transaction
from plm_engine.electrum.client import ElectrumEndpoint
from plm_engine.headers import header_meets_its_own_target
from plm_engine.network import REGTEST
from apps.lottery import bets, chain, confirmation
from apps.lottery.electrum_chain import ElectrumChain
from apps.lottery.models import AuditEntry, ChainTransaction, Movement, Participation, Utxo
from tests.django.test_chain_bets import ScriptedChain, bet, make_round, placed_and_signed, player, world # noqa: F401
pytestmark = pytest.mark.django_db
def pending(world):
user, round_, tx = placed_and_signed(world)
bets.broadcast_pending(ScriptedChain())
tx.refresh_from_db()
return user, round_, tx
# --- the write phase ------------------------------------------------------------------------------------------
def test_a_bet_is_confirmed_and_enters_the_draw(world):
user, round_, tx = pending(world)
assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 1
tx.refresh_from_db()
bet_row = Participation.objects.get()
assert tx.status == "confirmed" and tx.confirmed_at
assert (bet_row.status, bet_row.bet_txid) == ("confirmed", tx.current_txid) and bet_row.confirmed_at
assert Movement.objects.get(kind="bet").status == "confirmed"
assert set(Utxo.objects.filter(reserved_by=tx).values_list("spent_txid", flat=True)) == {tx.current_txid}
entry = AuditEntry.objects.get(action="tx_confirmed")
assert entry.payload["height"] == 120 and entry.payload["adopted"] is False
def test_an_earlier_version_that_confirms_is_adopted(world):
"""TX-31: the original won while a replacement was in flight."""
user, round_, tx = pending(world)
original = tx.current_txid
replacement = "ee" * 32
chain.register_version(tx, replacement, "00aa") # the row follows the replacement...
Participation.objects.update(bet_txid=replacement)
assert confirmation.apply_confirmations({tx.pk: (original, 130)}) == 1 # ...but the original is what the chain mined
tx.refresh_from_db()
assert tx.current_txid == original and tx.raw_hex and tx.raw_hex != "00aa"
assert Participation.objects.get().bet_txid == original
assert Utxo.objects.get(reserved_by=tx).spent_txid == original
assert AuditEntry.objects.get(action="tx_confirmed").payload["adopted"] is True
def test_a_row_that_is_no_longer_pending_is_left_alone(world):
user, round_, tx = pending(world)
ChainTransaction.objects.filter(pk=tx.pk).update(status="failed", failed_at=timezone.now(), failure_reason="x")
assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 0
assert Participation.objects.get().status == "pending"
def test_confirming_twice_changes_nothing_the_second_time(world):
user, round_, tx = pending(world)
confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)})
assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 0
assert AuditEntry.objects.filter(action="tx_confirmed").count() == 1
def test_the_plan_lists_pending_rows_with_every_version(world):
user, round_, tx = pending(world)
chain.register_version(tx, "ee" * 32, "00aa")
(row,) = confirmation.plan()
assert row[0] == tx.pk and row[1] == chain.assign_deposit_address(user).address and len(row[2]) == 2
def test_a_building_row_is_not_polled(world):
user, round_, tx = placed_and_signed(world) # signed, never accepted: still building
assert confirmation.plan() == []
# --- the network phase --------------------------------------------------------------------------------------------
def mined(root: bytes) -> str:
for nonce in range(10_000):
header = (struct.pack("<I", 1) + bytes(32) + root + struct.pack("<II", 0, 0x207FFFFF) + struct.pack("<I", nonce)).hex()
if header_meets_its_own_target(header, REGTEST.pow_limit):
return header
raise AssertionError
class Node:
def __init__(self, name, history, header, proof_ok=True, unreadable=()):
self.name, self.history, self.header, self.proof_ok, self.unreadable = name, history, header, proof_ok, set(unreadable)
class Client:
def __init__(self, node):
self.n = node
async def connect(self): pass
async def close(self): pass
async def get_history(self, scripthash):
if scripthash in self.n.unreadable:
raise OSError("timeout")
return self.n.history.get(scripthash, [])
async def get_block_header(self, height):
return self.n.header
async def get_merkle(self, txid, height):
return {"block_height": height, "merkle": [], "pos": 0} if self.n.proof_ok else {"block_height": height, "merkle": ["00" * 32], "pos": 0}
def network(nodes):
by = {ElectrumEndpoint(n.name, 1, False): n for n in nodes}
return ElectrumChain(list(by), REGTEST, False, lambda e: Client(by[e]))
TXID = "ab" * 32
HEADER = mined(bytes.fromhex(TXID)[::-1])
ADDRESS = "rplm1qz3r3xc0dxx470c9dhv78dmvlr0cd34vw3vvmsx"
def scripthash():
from plm_engine.electrum.scripthash import address_to_scripthash
return address_to_scripthash(ADDRESS)
def nodes(history, **kw):
return [Node(n, {scripthash(): history}, HEADER, **kw) for n in "abc"]
async def test_a_version_in_a_block_with_a_proof_is_confirmed():
wanted = [(7, ADDRESS, frozenset({TXID}))]
assert await network(nodes([{"tx_hash": TXID, "height": 150}])).observe_confirmations(wanted) == {7: (TXID, 150)}
async def test_a_non_current_version_is_found_too():
wanted = [(7, ADDRESS, frozenset({"cd" * 32, TXID}))] # the row follows the other one
assert (await network(nodes([{"tx_hash": TXID, "height": 150}])).observe_confirmations(wanted))[7][0] == TXID
@pytest.mark.parametrize("entry", [{"tx_hash": TXID, "height": 0}, {"tx_hash": TXID, "height": -1}, {"tx_hash": "11" * 32, "height": 150}])
async def test_the_mempool_and_other_transactions_are_not_confirmations(entry):
assert await network(nodes([entry])).observe_confirmations([(7, ADDRESS, frozenset({TXID}))]) == {}
async def test_without_a_proof_the_others_agree_on_nothing_is_confirmed():
assert await network(nodes([{"tx_hash": TXID, "height": 150}], proof_ok=False)).observe_confirmations(
[(7, ADDRESS, frozenset({TXID}))]) == {}
async def test_an_address_that_cannot_be_read_does_not_block_the_others():
other = "rplm1q3k3umjv83hhd7rvfafwfavs92na265gmude8xu"
from plm_engine.electrum.scripthash import address_to_scripthash
bad = address_to_scripthash(other)
servers = [Node(n, {scripthash(): [{"tx_hash": TXID, "height": 150}]}, HEADER, unreadable=[bad]) for n in "abc"]
wanted = [(1, other, frozenset({"cd" * 32})), (2, ADDRESS, frozenset({TXID}))]
assert await network(servers).observe_confirmations(wanted) == {2: (TXID, 150)}
async def test_with_no_server_nothing_is_confirmed():
class Down(Client):
async def connect(self):
raise OSError("down")
chain_ = ElectrumChain([ElectrumEndpoint("a", 1, False)], REGTEST, False, lambda e: Down(None))
assert await chain_.observe_confirmations([(1, ADDRESS, frozenset({TXID}))]) == {}
+181
View File
@@ -0,0 +1,181 @@
"""B7 — replace-by-fee: TX-32, TX-10..TX-13, TX-24, DAT-85."""
from datetime import timedelta
import pytest
from django.utils import timezone
from embit.transaction import Transaction
from plm_engine.psbt_builder import DUST_LIMIT_SATS, MAX_FEE_RATE_SAT_VB, estimate_vsize
from apps.lottery import bets, rbf
from apps.lottery.electrum_chain import Outcome, Verdict
from apps.lottery.models import AuditEntry, ChainTransaction, LotteryConfig, Participation, TransactionVersion
from tests.django.test_chain_bets import BET, PLM, ScriptedChain, bet, fund, make_round, player, world # noqa: F401
pytestmark = pytest.mark.django_db
def pending(world, funds=3 * BET):
user, round_ = player(funds=funds), make_round()
bet(user, round_)
bets.sign_pending()
bets.broadcast_pending(ScriptedChain())
return user, round_, ChainTransaction.objects.get()
def age(tx, seconds):
ChainTransaction.objects.filter(pk=tx.pk).update(last_broadcast_at=timezone.now() - timedelta(seconds=seconds))
# --- when ------------------------------------------------------------------------------------------------------
def test_a_fresh_transaction_is_not_due(world):
user, round_, tx = pending(world)
assert rbf.due() == []
age(tx, 899)
assert rbf.due() == []
age(tx, 901)
assert [t.pk for t in rbf.due()] == [tx.pk]
def test_the_timeout_is_the_configured_one(world):
user, round_, tx = pending(world)
config = LotteryConfig.get()
config.rbf_timeout_seconds = 60
config.save()
age(tx, 61)
assert [t.pk for t in rbf.due()] == [tx.pk]
def test_only_pending_bets_are_due(world):
user, round_ = player(), make_round()
bet(user, round_)
bets.sign_pending() # still building
ChainTransaction.objects.update(last_broadcast_at=timezone.now() - timedelta(days=1))
assert rbf.due() == []
# --- the replacement ---------------------------------------------------------------------------------------------
def test_a_slow_transaction_is_replaced_with_a_higher_fee(world):
user, round_, tx = pending(world)
old = Transaction.parse(bytes.fromhex(tx.raw_hex))
age(tx, 1000)
sent = ScriptedChain()
new_txid = rbf.bump(tx.pk, sent)
tx.refresh_from_db()
new = Transaction.parse(bytes.fromhex(tx.raw_hex))
assert new_txid == tx.current_txid != old.txid().hex() and sent.sent == [new_txid]
assert [(v.txid, v.vout) for v in new.vin] == [(v.txid, v.vout) for v in old.vin] # same coins
assert new.vout[0].value == old.vout[0].value and new.vout[0].script_pubkey.data == old.vout[0].script_pubkey.data
fee_old = 3 * BET - sum(o.value for o in old.vout) - 0 # inputs - outputs
fee_new = 3 * BET - sum(o.value for o in new.vout)
assert fee_new - fee_old >= estimate_vsize(1, 2) # BIP125: at least 1 sat/vB more
assert (tx.fee_rate_sat_vb, tx.attempt_count) == (2, 2) and tx.last_broadcast_at > timezone.now() - timedelta(seconds=30)
assert tx.versions.count() == 2 and Participation.objects.get().bet_txid == new_txid
assert AuditEntry.objects.filter(action="rbf_bumped").count() == 1
def test_the_replacement_is_recorded_before_it_is_sent(world):
user, round_, tx = pending(world)
old_txid = tx.current_txid
age(tx, 1000)
class Crashing:
async def broadcast(self, raw, txid):
raise RuntimeError("the process died here")
with pytest.raises(RuntimeError):
rbf.bump(tx.pk, Crashing())
tx.refresh_from_db()
assert tx.current_txid == old_txid and tx.versions.count() == 2 # the candidate is known, the row not moved
def test_an_uncertain_send_still_moves_the_row(world):
"""TX-12: the replacement may well be on the network, so the row follows it."""
user, round_, tx = pending(world)
age(tx, 1000)
assert rbf.bump(tx.pk, ScriptedChain(Verdict(Outcome.UNCERTAIN, "timeout"))) is not None
tx.refresh_from_db()
assert tx.attempt_count == 2 and tx.versions.count() == 2
def test_a_refused_replacement_leaves_the_row_and_is_recorded_once(world):
user, round_, tx = pending(world)
old_txid = tx.current_txid
age(tx, 1000)
refusal = Verdict(Outcome.REFUSED, "insufficient fee")
assert rbf.bump(tx.pk, ScriptedChain(refusal)) is None
assert rbf.bump(tx.pk, ScriptedChain(refusal)) is None
tx.refresh_from_db()
assert tx.current_txid == old_txid and tx.status == "pending"
assert AuditEntry.objects.filter(action="rbf_bump_failed").count() == 1 # TX-24: once per (txid, reason)
def test_a_row_that_changed_meanwhile_is_a_conflict_not_an_overwrite(world, monkeypatch):
"""TX-13: confirmed while the replacement was being sent. The row is not overwritten, the candidate stays."""
import asyncio
user, round_, tx = pending(world)
age(tx, 1000)
real_run = asyncio.run
def run_then_confirm(coroutine):
result = real_run(coroutine) # the send happens...
ChainTransaction.objects.filter(pk=tx.pk).update(status="confirmed", confirmed_at=timezone.now()) # ...then this
return result
monkeypatch.setattr(rbf.asyncio, "run", run_then_confirm)
assert rbf.bump(tx.pk, ScriptedChain()) is None
tx.refresh_from_db()
assert tx.status == "confirmed" and tx.attempt_count == 1 and tx.versions.count() == 2
def test_a_transaction_that_is_no_longer_pending_is_skipped(world):
user, round_, tx = pending(world)
ChainTransaction.objects.filter(pk=tx.pk).update(status="confirmed", confirmed_at=timezone.now())
assert rbf.bump(tx.pk, ScriptedChain()) is None and tx.versions.count() == 1
# --- limits ------------------------------------------------------------------------------------------------------
def test_the_maximum_fee_rate_is_never_exceeded(world):
user, round_, tx = pending(world)
ChainTransaction.objects.filter(pk=tx.pk).update(fee_rate_sat_vb=MAX_FEE_RATE_SAT_VB)
assert rbf.bump(tx.pk, ScriptedChain()) is None
assert AuditEntry.objects.get(action="rbf_bump_failed").payload["reason"] == "max_fee_rate"
def test_a_change_that_would_fall_below_dust_is_dropped_and_that_is_the_last_bump(world):
user, round_, tx = pending(world, funds=BET + 300) # a change of 300 sat
age(tx, 1000)
assert rbf.bump(tx.pk, ScriptedChain()) is not None
tx.refresh_from_db()
assert len(Transaction.parse(bytes.fromhex(tx.raw_hex)).vout) == 1 # only the pool output is left
age(tx, 1000)
assert rbf.bump(tx.pk, ScriptedChain()) is None # nothing left to take the increase from
assert AuditEntry.objects.get(action="rbf_bump_failed").payload["reason"] == "no_change_output"
def test_an_error_on_one_row_does_not_stop_the_others(world):
user, round_, first = pending(world)
other, other_round = player("luigi"), round_
bet(other, other_round)
bets.sign_pending()
bets.broadcast_pending(ScriptedChain())
for t in ChainTransaction.objects.all():
age(t, 1000)
ChainTransaction.objects.filter(pk=first.pk).update(fee_rate_sat_vb=MAX_FEE_RATE_SAT_VB) # this one cannot
assert rbf.bump_due(ScriptedChain()) == 1
def test_without_the_signing_key_nothing_is_bumped(world):
user, round_, tx = pending(world)
age(tx, 1000)
world.PROCESS_ROLE = "web"
assert rbf.bump(tx.pk, ScriptedChain()) is None
tx.refresh_from_db()
assert tx.versions.count() == 1 and tx.attempt_count == 1
+166
View File
@@ -0,0 +1,166 @@
"""B7 — reconciliation: TX-33, TX-15..TX-17, TX-23, TX-26."""
from datetime import timedelta
import pytest
from django.utils import timezone
from apps.lottery import bets, reconcile, reservation
from apps.lottery.balances import balance_of
from apps.lottery.electrum_chain import Outcome, Verdict
from apps.lottery.models import AuditEntry, ChainTransaction, LotteryConfig, Movement, Participation, Utxo
from tests.django.test_chain_bets import BET, ScriptedChain, bet, make_round, placed_and_signed, player, world # noqa: F401
pytestmark = pytest.mark.django_db
class Net:
"""The network as the reconciler asks about it."""
def __init__(self, known=(), complete=True, verdict=None, confirmed=None):
self.known, self.complete, self.verdict, self.confirmed = set(known), complete, verdict, confirmed or {}
self.sent = []
async def versions_known(self, address, txids):
return {t for t in txids if t in self.known}, self.complete
async def broadcast(self, raw, txid):
self.sent.append(txid)
return self.verdict or Verdict(Outcome.UNCERTAIN, "no answer")
async def observe_confirmations(self, wanted):
return {row: self.confirmed[row] for row, _a, _t in wanted if row in self.confirmed}
def old(tx, *, building_for=300, pending_for=None):
stamp = timezone.now() - timedelta(seconds=building_for)
ChainTransaction.objects.filter(pk=tx.pk).update(created_at=stamp)
if pending_for:
ChainTransaction.objects.filter(pk=tx.pk).update(broadcast_at=timezone.now() - timedelta(seconds=pending_for))
def building(world):
user, round_, tx = placed_and_signed(world)
old(tx)
return user, round_, tx
MISSING = "bad-txns-inputs-missingorspent"
# --- presence -----------------------------------------------------------------------------------------------------
def test_a_row_the_network_knows_is_promoted_to_pending(world):
user, round_, tx = building(world)
assert reconcile.reconcile_once(Net(known={tx.current_txid}))["recovered"] == 1
tx.refresh_from_db()
assert tx.status == "pending" and tx.broadcast_at
assert AuditEntry.objects.filter(action="pending_tx_recovered").count() == 1
def test_a_known_non_current_version_also_means_alive(world):
"""TX-15: any version known to any server is enough, not only the current one."""
from apps.lottery.chain import register_version
user, round_, tx = building(world)
earlier = tx.current_txid
register_version(tx, "ee" * 32, "00aa") # the row now follows a replacement the network has not seen
assert reconcile.reconcile_once(Net(known={earlier}))["recovered"] == 1
assert ChainTransaction.objects.get().status == "pending"
def test_a_fresh_building_row_is_not_examined_yet(world):
user, round_, tx = placed_and_signed(world) # created just now: inside the 120 s grace
net = Net(known=set())
assert reconcile.reconcile_once(net) == {"recovered": 0, "abandoned": 0, "resurrected": 0} and net.sent == []
def test_a_pending_row_is_examined_only_after_six_hours(world):
user, round_, tx = building(world)
reconcile._promote(tx.pk, timezone.now(), tx.current_txid)
old(tx, pending_for=5 * 3600)
net = Net()
reconcile.reconcile_once(net)
assert net.sent == []
old(tx, pending_for=7 * 3600)
reconcile.reconcile_once(net)
assert net.sent # absent and complete: the newest version is sent again
# --- abandonment only with proof ---------------------------------------------------------------------------------
def test_missing_inputs_with_no_server_knowing_it_abandons_and_frees_the_coins(world):
user, round_, tx = building(world)
net = Net(verdict=Verdict(Outcome.REFUSED, MISSING))
assert reconcile.reconcile_once(net)["abandoned"] == 1
tx.refresh_from_db()
assert tx.status == "failed" and MISSING in tx.failure_reason and tx.failed_at
assert Participation.objects.get().status == "rejected" and Movement.objects.get(kind="bet").status == "failed"
assert reservation.spendable(user).count() == 1 and balance_of(user).spendable_sats == 3 * BET
entry = AuditEntry.objects.get(action="tx_abandoned")
assert entry.payload["coins_freed"] == 1
@pytest.mark.parametrize("verdict", [
Verdict(Outcome.REFUSED, "insufficient fee, rejecting replacement"), # a refusal for the fee is not a proof
Verdict(Outcome.UNCERTAIN, "timeout"),
])
def test_anything_but_missing_inputs_abandons_nothing(world, verdict):
user, round_, tx = building(world)
reconcile.reconcile_once(Net(verdict=verdict))
tx.refresh_from_db()
assert tx.status in ("building", "pending") and tx.reserved_utxos.count() == 1 and not reservation.spendable(user).exists()
def test_an_accepted_resend_means_it_is_alive(world):
user, round_, tx = building(world)
assert reconcile.reconcile_once(Net(verdict=Verdict(Outcome.ACCEPTED, "")))["recovered"] == 1
def test_a_network_that_did_not_answer_enough_proves_nothing(world):
user, round_, tx = building(world)
net = Net(complete=False, verdict=Verdict(Outcome.REFUSED, MISSING))
assert reconcile.reconcile_once(net) == {"recovered": 0, "abandoned": 0, "resurrected": 0} and net.sent == []
assert ChainTransaction.objects.get().status == "building"
@pytest.mark.parametrize("reason,expected", [
("bad-txns-inputs-missingorspent", True), ("missing-inputs", True), ("the transaction was rejected\n\nmissing inputs", True),
("insufficient fee", False), ("mempool min fee not met", False), ("", False),
])
def test_only_missing_or_spent_inputs_count_as_proof(reason, expected):
assert reconcile.is_missing_inputs(reason) is expected
def test_a_row_that_changed_meanwhile_is_not_overwritten(world):
user, round_, tx = building(world)
ChainTransaction.objects.filter(pk=tx.pk).update(status="pending") # promoted by someone else
assert reconcile._promote(tx.pk, timezone.now(), tx.current_txid) is False
# --- failed rows stay watched (TX-23) -----------------------------------------------------------------------------
def failed(world):
user, round_, tx = building(world)
reconcile.reconcile_once(Net(verdict=Verdict(Outcome.REFUSED, MISSING)))
tx.refresh_from_db()
return tx
def test_a_failed_row_that_confirms_is_reported_once_and_pauses_new_bets(world):
tx = failed(world)
net = Net(confirmed={tx.pk: (tx.current_txid, 200)})
assert reconcile.reconcile_once(net)["resurrected"] == 1
assert reconcile.reconcile_once(net)["resurrected"] == 0 # once
assert AuditEntry.objects.filter(action="tx_resurrected").count() == 1
assert LotteryConfig.get().new_bets_suspended
assert ChainTransaction.objects.get().status == "failed" # the reconciler does not fix the linked rows itself
def test_a_failed_row_is_watched_for_fourteen_days_only(world):
tx = failed(world)
ChainTransaction.objects.filter(pk=tx.pk).update(failed_at=timezone.now() - timedelta(days=15))
assert reconcile.reconcile_once(Net(confirmed={tx.pk: (tx.current_txid, 200)}))["resurrected"] == 0
+1 -1
View File
@@ -222,7 +222,7 @@ def test_the_operator_changes_the_threshold_from_the_admin_and_it_is_audited(cli
client.force_login(admin)
config = LotteryConfig.get()
data = {f: getattr(config, f) for f in ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds",
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb")}
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb", "rbf_timeout_seconds")}
data["deposit_confirmations"] = 3
assert client.post("/admin/lottery/lotteryconfig/1/change/", data).status_code == 302
assert LotteryConfig.get().deposit_confirmations == 3
@@ -0,0 +1,84 @@
"""B7 — confirmation, replace-by-fee and the adoption of an earlier version, on the local regtest
(TX-30..TX-32, TX-12, TX-19)."""
import time
from datetime import timedelta
import pytest
from django.utils import timezone
from apps.lottery import bets, confirmation, rbf
from apps.lottery.ledger import ChainLedger
from apps.lottery.models import ChainTransaction, Movement, Participation, Utxo
from plm_engine.regtest import RegtestError
from tests.django.test_chain_bets import ScriptedChain
from tests.integration.test_bet_regtest import BET, PLM, funded_player, network, open_round # noqa: F401
from uuid import uuid4
pytestmark = pytest.mark.django_db
def placed(user, network):
ChainLedger().place_bet(user, open_round().pk, request_id=uuid4())
assert bets.process_bets(network)["accepted"] == 1
return ChainTransaction.objects.get()
def poll_until(network, tx, status="confirmed"):
for _ in range(80):
confirmation.poll_confirmations(network)
tx.refresh_from_db()
if tx.status == status:
return
time.sleep(0.25)
def age(tx):
ChainTransaction.objects.filter(pk=tx.pk).update(last_broadcast_at=timezone.now() - timedelta(hours=1))
def test_a_bet_is_confirmed_by_a_block_and_enters_the_draw(regtest_node, network, funded_player):
user, _ = funded_player
tx = placed(user, network)
regtest_node.mine(1)
poll_until(network, tx)
assert tx.status == "confirmed"
bet = Participation.objects.get()
assert bet.status == "confirmed" and bet.confirmed_at and bet.bet_txid == tx.current_txid
assert Movement.objects.get(kind="bet").status == "confirmed"
assert Utxo.objects.get(reserved_by=tx).spent_txid == tx.current_txid
def test_a_slow_bet_is_replaced_by_fee_and_the_replacement_confirms(regtest_node, network, funded_player):
user, _ = funded_player
tx = placed(user, network)
first = tx.current_txid
age(tx)
assert rbf.bump_due(network) == 1
tx.refresh_from_db()
assert tx.current_txid != first and tx.fee_rate_sat_vb == 2
assert regtest_node.rpc("getmempoolentry", tx.current_txid)["vsize"] > 0 # the node took the replacement...
with pytest.raises(RegtestError):
regtest_node.rpc("getmempoolentry", first) # ...and dropped the original
regtest_node.mine(1)
poll_until(network, tx)
assert tx.status == "confirmed" and Participation.objects.get().bet_txid == tx.current_txid
assert tx.versions.count() == 2
def test_an_earlier_version_that_confirms_is_adopted(regtest_node, network, funded_player):
"""The replacement never reached the node (the send was uncertain); the original is what the chain mines."""
user, _ = funded_player
tx = placed(user, network)
original = tx.current_txid
age(tx)
from apps.lottery.electrum_chain import Outcome, Verdict
assert rbf.bump(tx.pk, ScriptedChain(Verdict(Outcome.UNCERTAIN, "timeout"))) is not None # the row follows version 2
tx.refresh_from_db()
assert tx.current_txid != original
regtest_node.mine(1) # version 1 is in the node's mempool and gets mined
poll_until(network, tx)
assert tx.status == "confirmed" and tx.current_txid == original # adopted (TX-31)
assert Participation.objects.get().bet_txid == original
assert Utxo.objects.get(reserved_by=tx).spent_txid == original