B7: confirmations with merkle proof, adoption of the confirmed version, RBF bumper and reconciler (TX-30..33, DAT-85)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
davideandClaude Sonnet 5.5 committed 2026-11-16 16:29:08 +01:00
1 parent feff993584
commit fa57d2e5ab
20 files changed
+1165 -20

No files matched your search

+176
View File
@@ -0,0 +1,176 @@
"""B7 — confirmations: TX-30, TX-31, TX-26, TX-08."""
import struct
from datetime import timedelta
import pytest
from django.utils import timezone
from embit.transaction import Transaction
from plm_engine.electrum.client import ElectrumEndpoint
from plm_engine.headers import header_meets_its_own_target
from plm_engine.network import REGTEST
from apps.lottery import bets, chain, confirmation
from apps.lottery.electrum_chain import ElectrumChain
from apps.lottery.models import AuditEntry, ChainTransaction, Movement, Participation, Utxo
from tests.django.test_chain_bets import ScriptedChain, bet, make_round, placed_and_signed, player, world # noqa: F401
pytestmark = pytest.mark.django_db
def pending(world):
user, round_, tx = placed_and_signed(world)
bets.broadcast_pending(ScriptedChain())
tx.refresh_from_db()
return user, round_, tx
# --- the write phase ------------------------------------------------------------------------------------------
def test_a_bet_is_confirmed_and_enters_the_draw(world):
user, round_, tx = pending(world)
assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 1
tx.refresh_from_db()
bet_row = Participation.objects.get()
assert tx.status == "confirmed" and tx.confirmed_at
assert (bet_row.status, bet_row.bet_txid) == ("confirmed", tx.current_txid) and bet_row.confirmed_at
assert Movement.objects.get(kind="bet").status == "confirmed"
assert set(Utxo.objects.filter(reserved_by=tx).values_list("spent_txid", flat=True)) == {tx.current_txid}
entry = AuditEntry.objects.get(action="tx_confirmed")
assert entry.payload["height"] == 120 and entry.payload["adopted"] is False
def test_an_earlier_version_that_confirms_is_adopted(world):
"""TX-31: the original won while a replacement was in flight."""
user, round_, tx = pending(world)
original = tx.current_txid
replacement = "ee" * 32
chain.register_version(tx, replacement, "00aa") # the row follows the replacement...
Participation.objects.update(bet_txid=replacement)
assert confirmation.apply_confirmations({tx.pk: (original, 130)}) == 1 # ...but the original is what the chain mined
tx.refresh_from_db()
assert tx.current_txid == original and tx.raw_hex and tx.raw_hex != "00aa"
assert Participation.objects.get().bet_txid == original
assert Utxo.objects.get(reserved_by=tx).spent_txid == original
assert AuditEntry.objects.get(action="tx_confirmed").payload["adopted"] is True
def test_a_row_that_is_no_longer_pending_is_left_alone(world):
user, round_, tx = pending(world)
ChainTransaction.objects.filter(pk=tx.pk).update(status="failed", failed_at=timezone.now(), failure_reason="x")
assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 0
assert Participation.objects.get().status == "pending"
def test_confirming_twice_changes_nothing_the_second_time(world):
user, round_, tx = pending(world)
confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)})
assert confirmation.apply_confirmations({tx.pk: (tx.current_txid, 120)}) == 0
assert AuditEntry.objects.filter(action="tx_confirmed").count() == 1
def test_the_plan_lists_pending_rows_with_every_version(world):
user, round_, tx = pending(world)
chain.register_version(tx, "ee" * 32, "00aa")
(row,) = confirmation.plan()
assert row[0] == tx.pk and row[1] == chain.assign_deposit_address(user).address and len(row[2]) == 2
def test_a_building_row_is_not_polled(world):
user, round_, tx = placed_and_signed(world) # signed, never accepted: still building
assert confirmation.plan() == []
# --- the network phase --------------------------------------------------------------------------------------------
def mined(root: bytes) -> str:
for nonce in range(10_000):
header = (struct.pack("<I", 1) + bytes(32) + root + struct.pack("<II", 0, 0x207FFFFF) + struct.pack("<I", nonce)).hex()
if header_meets_its_own_target(header, REGTEST.pow_limit):
return header
raise AssertionError
class Node:
def __init__(self, name, history, header, proof_ok=True, unreadable=()):
self.name, self.history, self.header, self.proof_ok, self.unreadable = name, history, header, proof_ok, set(unreadable)
class Client:
def __init__(self, node):
self.n = node
async def connect(self): pass
async def close(self): pass
async def get_history(self, scripthash):
if scripthash in self.n.unreadable:
raise OSError("timeout")
return self.n.history.get(scripthash, [])
async def get_block_header(self, height):
return self.n.header
async def get_merkle(self, txid, height):
return {"block_height": height, "merkle": [], "pos": 0} if self.n.proof_ok else {"block_height": height, "merkle": ["00" * 32], "pos": 0}
def network(nodes):
by = {ElectrumEndpoint(n.name, 1, False): n for n in nodes}
return ElectrumChain(list(by), REGTEST, False, lambda e: Client(by[e]))
TXID = "ab" * 32
HEADER = mined(bytes.fromhex(TXID)[::-1])
ADDRESS = "rplm1qz3r3xc0dxx470c9dhv78dmvlr0cd34vw3vvmsx"
def scripthash():
from plm_engine.electrum.scripthash import address_to_scripthash
return address_to_scripthash(ADDRESS)
def nodes(history, **kw):
return [Node(n, {scripthash(): history}, HEADER, **kw) for n in "abc"]
async def test_a_version_in_a_block_with_a_proof_is_confirmed():
wanted = [(7, ADDRESS, frozenset({TXID}))]
assert await network(nodes([{"tx_hash": TXID, "height": 150}])).observe_confirmations(wanted) == {7: (TXID, 150)}
async def test_a_non_current_version_is_found_too():
wanted = [(7, ADDRESS, frozenset({"cd" * 32, TXID}))] # the row follows the other one
assert (await network(nodes([{"tx_hash": TXID, "height": 150}])).observe_confirmations(wanted))[7][0] == TXID
@pytest.mark.parametrize("entry", [{"tx_hash": TXID, "height": 0}, {"tx_hash": TXID, "height": -1}, {"tx_hash": "11" * 32, "height": 150}])
async def test_the_mempool_and_other_transactions_are_not_confirmations(entry):
assert await network(nodes([entry])).observe_confirmations([(7, ADDRESS, frozenset({TXID}))]) == {}
async def test_without_a_proof_the_others_agree_on_nothing_is_confirmed():
assert await network(nodes([{"tx_hash": TXID, "height": 150}], proof_ok=False)).observe_confirmations(
[(7, ADDRESS, frozenset({TXID}))]) == {}
async def test_an_address_that_cannot_be_read_does_not_block_the_others():
other = "rplm1q3k3umjv83hhd7rvfafwfavs92na265gmude8xu"
from plm_engine.electrum.scripthash import address_to_scripthash
bad = address_to_scripthash(other)
servers = [Node(n, {scripthash(): [{"tx_hash": TXID, "height": 150}]}, HEADER, unreadable=[bad]) for n in "abc"]
wanted = [(1, other, frozenset({"cd" * 32})), (2, ADDRESS, frozenset({TXID}))]
assert await network(servers).observe_confirmations(wanted) == {2: (TXID, 150)}
async def test_with_no_server_nothing_is_confirmed():
class Down(Client):
async def connect(self):
raise OSError("down")
chain_ = ElectrumChain([ElectrumEndpoint("a", 1, False)], REGTEST, False, lambda e: Down(None))
assert await chain_.observe_confirmations([(1, ADDRESS, frozenset({TXID}))]) == {}
+181
View File
@@ -0,0 +1,181 @@
"""B7 — replace-by-fee: TX-32, TX-10..TX-13, TX-24, DAT-85."""
from datetime import timedelta
import pytest
from django.utils import timezone
from embit.transaction import Transaction
from plm_engine.psbt_builder import DUST_LIMIT_SATS, MAX_FEE_RATE_SAT_VB, estimate_vsize
from apps.lottery import bets, rbf
from apps.lottery.electrum_chain import Outcome, Verdict
from apps.lottery.models import AuditEntry, ChainTransaction, LotteryConfig, Participation, TransactionVersion
from tests.django.test_chain_bets import BET, PLM, ScriptedChain, bet, fund, make_round, player, world # noqa: F401
pytestmark = pytest.mark.django_db
def pending(world, funds=3 * BET):
user, round_ = player(funds=funds), make_round()
bet(user, round_)
bets.sign_pending()
bets.broadcast_pending(ScriptedChain())
return user, round_, ChainTransaction.objects.get()
def age(tx, seconds):
ChainTransaction.objects.filter(pk=tx.pk).update(last_broadcast_at=timezone.now() - timedelta(seconds=seconds))
# --- when ------------------------------------------------------------------------------------------------------
def test_a_fresh_transaction_is_not_due(world):
user, round_, tx = pending(world)
assert rbf.due() == []
age(tx, 899)
assert rbf.due() == []
age(tx, 901)
assert [t.pk for t in rbf.due()] == [tx.pk]
def test_the_timeout_is_the_configured_one(world):
user, round_, tx = pending(world)
config = LotteryConfig.get()
config.rbf_timeout_seconds = 60
config.save()
age(tx, 61)
assert [t.pk for t in rbf.due()] == [tx.pk]
def test_only_pending_bets_are_due(world):
user, round_ = player(), make_round()
bet(user, round_)
bets.sign_pending() # still building
ChainTransaction.objects.update(last_broadcast_at=timezone.now() - timedelta(days=1))
assert rbf.due() == []
# --- the replacement ---------------------------------------------------------------------------------------------
def test_a_slow_transaction_is_replaced_with_a_higher_fee(world):
user, round_, tx = pending(world)
old = Transaction.parse(bytes.fromhex(tx.raw_hex))
age(tx, 1000)
sent = ScriptedChain()
new_txid = rbf.bump(tx.pk, sent)
tx.refresh_from_db()
new = Transaction.parse(bytes.fromhex(tx.raw_hex))
assert new_txid == tx.current_txid != old.txid().hex() and sent.sent == [new_txid]
assert [(v.txid, v.vout) for v in new.vin] == [(v.txid, v.vout) for v in old.vin] # same coins
assert new.vout[0].value == old.vout[0].value and new.vout[0].script_pubkey.data == old.vout[0].script_pubkey.data
fee_old = 3 * BET - sum(o.value for o in old.vout) - 0 # inputs - outputs
fee_new = 3 * BET - sum(o.value for o in new.vout)
assert fee_new - fee_old >= estimate_vsize(1, 2) # BIP125: at least 1 sat/vB more
assert (tx.fee_rate_sat_vb, tx.attempt_count) == (2, 2) and tx.last_broadcast_at > timezone.now() - timedelta(seconds=30)
assert tx.versions.count() == 2 and Participation.objects.get().bet_txid == new_txid
assert AuditEntry.objects.filter(action="rbf_bumped").count() == 1
def test_the_replacement_is_recorded_before_it_is_sent(world):
user, round_, tx = pending(world)
old_txid = tx.current_txid
age(tx, 1000)
class Crashing:
async def broadcast(self, raw, txid):
raise RuntimeError("the process died here")
with pytest.raises(RuntimeError):
rbf.bump(tx.pk, Crashing())
tx.refresh_from_db()
assert tx.current_txid == old_txid and tx.versions.count() == 2 # the candidate is known, the row not moved
def test_an_uncertain_send_still_moves_the_row(world):
"""TX-12: the replacement may well be on the network, so the row follows it."""
user, round_, tx = pending(world)
age(tx, 1000)
assert rbf.bump(tx.pk, ScriptedChain(Verdict(Outcome.UNCERTAIN, "timeout"))) is not None
tx.refresh_from_db()
assert tx.attempt_count == 2 and tx.versions.count() == 2
def test_a_refused_replacement_leaves_the_row_and_is_recorded_once(world):
user, round_, tx = pending(world)
old_txid = tx.current_txid
age(tx, 1000)
refusal = Verdict(Outcome.REFUSED, "insufficient fee")
assert rbf.bump(tx.pk, ScriptedChain(refusal)) is None
assert rbf.bump(tx.pk, ScriptedChain(refusal)) is None
tx.refresh_from_db()
assert tx.current_txid == old_txid and tx.status == "pending"
assert AuditEntry.objects.filter(action="rbf_bump_failed").count() == 1 # TX-24: once per (txid, reason)
def test_a_row_that_changed_meanwhile_is_a_conflict_not_an_overwrite(world, monkeypatch):
"""TX-13: confirmed while the replacement was being sent. The row is not overwritten, the candidate stays."""
import asyncio
user, round_, tx = pending(world)
age(tx, 1000)
real_run = asyncio.run
def run_then_confirm(coroutine):
result = real_run(coroutine) # the send happens...
ChainTransaction.objects.filter(pk=tx.pk).update(status="confirmed", confirmed_at=timezone.now()) # ...then this
return result
monkeypatch.setattr(rbf.asyncio, "run", run_then_confirm)
assert rbf.bump(tx.pk, ScriptedChain()) is None
tx.refresh_from_db()
assert tx.status == "confirmed" and tx.attempt_count == 1 and tx.versions.count() == 2
def test_a_transaction_that_is_no_longer_pending_is_skipped(world):
user, round_, tx = pending(world)
ChainTransaction.objects.filter(pk=tx.pk).update(status="confirmed", confirmed_at=timezone.now())
assert rbf.bump(tx.pk, ScriptedChain()) is None and tx.versions.count() == 1
# --- limits ------------------------------------------------------------------------------------------------------
def test_the_maximum_fee_rate_is_never_exceeded(world):
user, round_, tx = pending(world)
ChainTransaction.objects.filter(pk=tx.pk).update(fee_rate_sat_vb=MAX_FEE_RATE_SAT_VB)
assert rbf.bump(tx.pk, ScriptedChain()) is None
assert AuditEntry.objects.get(action="rbf_bump_failed").payload["reason"] == "max_fee_rate"
def test_a_change_that_would_fall_below_dust_is_dropped_and_that_is_the_last_bump(world):
user, round_, tx = pending(world, funds=BET + 300) # a change of 300 sat
age(tx, 1000)
assert rbf.bump(tx.pk, ScriptedChain()) is not None
tx.refresh_from_db()
assert len(Transaction.parse(bytes.fromhex(tx.raw_hex)).vout) == 1 # only the pool output is left
age(tx, 1000)
assert rbf.bump(tx.pk, ScriptedChain()) is None # nothing left to take the increase from
assert AuditEntry.objects.get(action="rbf_bump_failed").payload["reason"] == "no_change_output"
def test_an_error_on_one_row_does_not_stop_the_others(world):
user, round_, first = pending(world)
other, other_round = player("luigi"), round_
bet(other, other_round)
bets.sign_pending()
bets.broadcast_pending(ScriptedChain())
for t in ChainTransaction.objects.all():
age(t, 1000)
ChainTransaction.objects.filter(pk=first.pk).update(fee_rate_sat_vb=MAX_FEE_RATE_SAT_VB) # this one cannot
assert rbf.bump_due(ScriptedChain()) == 1
def test_without_the_signing_key_nothing_is_bumped(world):
user, round_, tx = pending(world)
age(tx, 1000)
world.PROCESS_ROLE = "web"
assert rbf.bump(tx.pk, ScriptedChain()) is None
tx.refresh_from_db()
assert tx.versions.count() == 1 and tx.attempt_count == 1
+166
View File
@@ -0,0 +1,166 @@
"""B7 — reconciliation: TX-33, TX-15..TX-17, TX-23, TX-26."""
from datetime import timedelta
import pytest
from django.utils import timezone
from apps.lottery import bets, reconcile, reservation
from apps.lottery.balances import balance_of
from apps.lottery.electrum_chain import Outcome, Verdict
from apps.lottery.models import AuditEntry, ChainTransaction, LotteryConfig, Movement, Participation, Utxo
from tests.django.test_chain_bets import BET, ScriptedChain, bet, make_round, placed_and_signed, player, world # noqa: F401
pytestmark = pytest.mark.django_db
class Net:
"""The network as the reconciler asks about it."""
def __init__(self, known=(), complete=True, verdict=None, confirmed=None):
self.known, self.complete, self.verdict, self.confirmed = set(known), complete, verdict, confirmed or {}
self.sent = []
async def versions_known(self, address, txids):
return {t for t in txids if t in self.known}, self.complete
async def broadcast(self, raw, txid):
self.sent.append(txid)
return self.verdict or Verdict(Outcome.UNCERTAIN, "no answer")
async def observe_confirmations(self, wanted):
return {row: self.confirmed[row] for row, _a, _t in wanted if row in self.confirmed}
def old(tx, *, building_for=300, pending_for=None):
stamp = timezone.now() - timedelta(seconds=building_for)
ChainTransaction.objects.filter(pk=tx.pk).update(created_at=stamp)
if pending_for:
ChainTransaction.objects.filter(pk=tx.pk).update(broadcast_at=timezone.now() - timedelta(seconds=pending_for))
def building(world):
user, round_, tx = placed_and_signed(world)
old(tx)
return user, round_, tx
MISSING = "bad-txns-inputs-missingorspent"
# --- presence -----------------------------------------------------------------------------------------------------
def test_a_row_the_network_knows_is_promoted_to_pending(world):
user, round_, tx = building(world)
assert reconcile.reconcile_once(Net(known={tx.current_txid}))["recovered"] == 1
tx.refresh_from_db()
assert tx.status == "pending" and tx.broadcast_at
assert AuditEntry.objects.filter(action="pending_tx_recovered").count() == 1
def test_a_known_non_current_version_also_means_alive(world):
"""TX-15: any version known to any server is enough, not only the current one."""
from apps.lottery.chain import register_version
user, round_, tx = building(world)
earlier = tx.current_txid
register_version(tx, "ee" * 32, "00aa") # the row now follows a replacement the network has not seen
assert reconcile.reconcile_once(Net(known={earlier}))["recovered"] == 1
assert ChainTransaction.objects.get().status == "pending"
def test_a_fresh_building_row_is_not_examined_yet(world):
user, round_, tx = placed_and_signed(world) # created just now: inside the 120 s grace
net = Net(known=set())
assert reconcile.reconcile_once(net) == {"recovered": 0, "abandoned": 0, "resurrected": 0} and net.sent == []
def test_a_pending_row_is_examined_only_after_six_hours(world):
user, round_, tx = building(world)
reconcile._promote(tx.pk, timezone.now(), tx.current_txid)
old(tx, pending_for=5 * 3600)
net = Net()
reconcile.reconcile_once(net)
assert net.sent == []
old(tx, pending_for=7 * 3600)
reconcile.reconcile_once(net)
assert net.sent # absent and complete: the newest version is sent again
# --- abandonment only with proof ---------------------------------------------------------------------------------
def test_missing_inputs_with_no_server_knowing_it_abandons_and_frees_the_coins(world):
user, round_, tx = building(world)
net = Net(verdict=Verdict(Outcome.REFUSED, MISSING))
assert reconcile.reconcile_once(net)["abandoned"] == 1
tx.refresh_from_db()
assert tx.status == "failed" and MISSING in tx.failure_reason and tx.failed_at
assert Participation.objects.get().status == "rejected" and Movement.objects.get(kind="bet").status == "failed"
assert reservation.spendable(user).count() == 1 and balance_of(user).spendable_sats == 3 * BET
entry = AuditEntry.objects.get(action="tx_abandoned")
assert entry.payload["coins_freed"] == 1
@pytest.mark.parametrize("verdict", [
Verdict(Outcome.REFUSED, "insufficient fee, rejecting replacement"), # a refusal for the fee is not a proof
Verdict(Outcome.UNCERTAIN, "timeout"),
])
def test_anything_but_missing_inputs_abandons_nothing(world, verdict):
user, round_, tx = building(world)
reconcile.reconcile_once(Net(verdict=verdict))
tx.refresh_from_db()
assert tx.status in ("building", "pending") and tx.reserved_utxos.count() == 1 and not reservation.spendable(user).exists()
def test_an_accepted_resend_means_it_is_alive(world):
user, round_, tx = building(world)
assert reconcile.reconcile_once(Net(verdict=Verdict(Outcome.ACCEPTED, "")))["recovered"] == 1
def test_a_network_that_did_not_answer_enough_proves_nothing(world):
user, round_, tx = building(world)
net = Net(complete=False, verdict=Verdict(Outcome.REFUSED, MISSING))
assert reconcile.reconcile_once(net) == {"recovered": 0, "abandoned": 0, "resurrected": 0} and net.sent == []
assert ChainTransaction.objects.get().status == "building"
@pytest.mark.parametrize("reason,expected", [
("bad-txns-inputs-missingorspent", True), ("missing-inputs", True), ("the transaction was rejected\n\nmissing inputs", True),
("insufficient fee", False), ("mempool min fee not met", False), ("", False),
])
def test_only_missing_or_spent_inputs_count_as_proof(reason, expected):
assert reconcile.is_missing_inputs(reason) is expected
def test_a_row_that_changed_meanwhile_is_not_overwritten(world):
user, round_, tx = building(world)
ChainTransaction.objects.filter(pk=tx.pk).update(status="pending") # promoted by someone else
assert reconcile._promote(tx.pk, timezone.now(), tx.current_txid) is False
# --- failed rows stay watched (TX-23) -----------------------------------------------------------------------------
def failed(world):
user, round_, tx = building(world)
reconcile.reconcile_once(Net(verdict=Verdict(Outcome.REFUSED, MISSING)))
tx.refresh_from_db()
return tx
def test_a_failed_row_that_confirms_is_reported_once_and_pauses_new_bets(world):
tx = failed(world)
net = Net(confirmed={tx.pk: (tx.current_txid, 200)})
assert reconcile.reconcile_once(net)["resurrected"] == 1
assert reconcile.reconcile_once(net)["resurrected"] == 0 # once
assert AuditEntry.objects.filter(action="tx_resurrected").count() == 1
assert LotteryConfig.get().new_bets_suspended
assert ChainTransaction.objects.get().status == "failed" # the reconciler does not fix the linked rows itself
def test_a_failed_row_is_watched_for_fourteen_days_only(world):
tx = failed(world)
ChainTransaction.objects.filter(pk=tx.pk).update(failed_at=timezone.now() - timedelta(days=15))
assert reconcile.reconcile_once(Net(confirmed={tx.pk: (tx.current_txid, 200)}))["resurrected"] == 0
+1 -1
View File
@@ -222,7 +222,7 @@ def test_the_operator_changes_the_threshold_from_the_admin_and_it_is_audited(cli
client.force_login(admin)
config = LotteryConfig.get()
data = {f: getattr(config, f) for f in ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds",
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb")}
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb", "rbf_timeout_seconds")}
data["deposit_confirmations"] = 3
assert client.post("/admin/lottery/lotteryconfig/1/change/", data).status_code == 302
assert LotteryConfig.get().deposit_confirmations == 3