diff --git a/.env.example b/.env.example index dde4afa..0f9bbca 100644 --- a/.env.example +++ b/.env.example @@ -70,8 +70,9 @@ DJANGO_SECRET_KEY= PLM_NETWORK=regtest PLM_MAINNET_CONFIRMED= # ARC-78: the web process knows only the PUBLIC account key. Create the keys once with -# docker compose run --rm worker python manage.py init_keys /app/db_data/keys (move them out of there afterwards) -# and paste the PLM_ACCOUNT_XPUB line it prints. The private files are for the worker alone (mounted from B6). +# docker compose run --rm worker python manage.py init_keys /run/plm-keys-new (then move the two files to ./data/keys) +# and paste the PLM_ACCOUNT_XPUB line it prints. The private files are for the worker alone: put them in ./data/keys, +# which only the worker mounts (read-only). PLM_ACCOUNT_XPUB= # Electrum servers of a real ledger (LEDGER_BACKEND=chain), first = preferred, at least three for a quorum: # host:port[:notls][:pin=],host:port,... diff --git a/CLAUDE.md b/CLAUDE.md index 9fa087b..a4af8b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ The user communicates in Italian in chat — reply to them in Italian. Everythin ## Project status -The migration from FastAPI to Django is in progress on branch `django-migration` (plan: [roadmap.md](roadmap.md), steps A1–A10, B1–B5 done). The suite has 1263 tests: the old stack's under `tests/unit/`, the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Open defects of the old stack are in [BUGS.md](BUGS.md); accepted limits in [docs/limiti-noti.md](docs/limiti-noti.md); every closed `B-nn` is in [docs/storico-bug.md](docs/storico-bug.md) (`git log --all --grep 'B-nn'` finds its commit). +The migration from FastAPI to Django is in progress on branch `django-migration` (plan: [roadmap.md](roadmap.md), steps A1–A10, B1–B6 done). The suite has 1288 tests: the old stack's under `tests/unit/`, the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Open defects of the old stack are in [BUGS.md](BUGS.md); accepted limits in [docs/limiti-noti.md](docs/limiti-noti.md); every closed `B-nn` is in [docs/storico-bug.md](docs/storico-bug.md) (`git log --all --grep 'B-nn'` finds its commit). **[docs/](docs/README.md) is the specification, and it is authoritative.** Every verifiable rule there has a stable requirement ID (`RND-04`, `BET-02`, …) with `*Codice:*`/`*Test:*` pointers to what implements it; the index is [docs/come-funziona.md](docs/come-funziona.md) and the ID convention and change workflow are in [docs/processo-di-modifica.md](docs/processo-di-modifica.md). A behaviour change starts in the spec: edit or add the requirement, implement it, cite the ID in the test docstring and the commit message, and ship spec + code + test in one commit. If this file and the spec disagree, the spec wins and this file is the one to fix. Before writing code, read the relevant spec documents and the "Architecture" section below. diff --git a/README.md b/README.md index 3a75e1a..afbb43c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ python -m pytest # all tests python -m pytest tests/unit/test_hd.py # one file ``` -1263 unit tests cover HD derivation, PSBT building, the Electrum client, bets, +1288 unit tests cover HD derivation, PSBT building, the Electrum client, bets, deposits, withdrawals, the round/draw engine, RBF fee-bumping, admin config, the pending-inclusive balance calculation, and the SSE push channel. No automated integration tests against a live Electrum connection — mainnet diff --git a/apps/lottery/admin.py b/apps/lottery/admin.py index ff69676..7d273d5 100644 --- a/apps/lottery/admin.py +++ b/apps/lottery/admin.py @@ -83,7 +83,7 @@ class LotteryConfigAdmin(admin.ModelAdmin): commands, which ask for a reason (WEB-43). A change is audited with what changed.""" fields = ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds", - "draw_confirmations", "deposit_confirmations", "max_round_pool_sats") + "draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb") def has_add_permission(self, request): return False diff --git a/apps/lottery/bets.py b/apps/lottery/bets.py new file mode 100644 index 0000000..be6c853 --- /dev/null +++ b/apps/lottery/bets.py @@ -0,0 +1,149 @@ +"""The worker's half of a real bet (roadmap B6, BET-31..BET-33). + +The web process admitted the bet and reserved its coins (`ChainLedger.place_bet`). Here, with the signing key +that only this process may hold (ARC-78): + +* `sign_pending` builds and signs the transaction and — in one database transaction — saves its version and the + bet's real amount to the pool, *before anything is sent*; +* `broadcast_pending` sends what was saved and writes the outcome, only if the row is still as it was read (TX-26). + Only a proven refusal frees coins; an uncertain answer changes nothing and the same transaction is sent again. +""" + +import asyncio +import logging +from datetime import datetime + +from django.conf import settings +from django.db import transaction +from django.utils import timezone +from embit import script +from plm_engine import hd +from plm_engine.psbt_builder import RECIPIENT_VOUT, InsufficientFundsError, Utxo as EngineUtxo, build_signed_transaction + +from . import audit, keys, reservation +from .chain import register_version +from .electrum_chain import Outcome +from .models import ChainTransaction, DepositAddress, LotteryConfig, Movement, Participation + +logger = logging.getLogger(__name__) +WORKER = "worker" +_BATCH = 20 + + +def _bet_of(tx: ChainTransaction) -> Participation | None: + return Participation.objects.filter(movements__chain_transaction=tx, movements__kind=Movement.Kind.BET).first() + + +def sign_pending() -> int: + """BET-31. Returns how many bets were signed and saved. Never sends anything.""" + todo = list(ChainTransaction.objects.filter( + kind=ChainTransaction.Kind.BET, status=ChainTransaction.Status.BUILDING, current_txid__isnull=True, + ).select_related("user", "round").order_by("id")[:_BATCH]) + if not todo: + return 0 + try: + account = keys.signing_account_key() + except keys.SigningKeyUnavailable as error: + logger.error("cannot sign %d pending bet(s): %s", len(todo), error) + return 0 + fee_rate = LotteryConfig.get().fee_rate_sat_vb + signed = 0 + for tx in todo: + try: + if _sign_one(tx, account, fee_rate): + signed += 1 + except InsufficientFundsError as error: + _fail(tx, f"cannot build the transaction: {error}", release=True) + except Exception: + logger.exception("signing bet transaction %s failed; it stays building", tx.pk) + return signed + + +def _sign_one(tx: ChainTransaction, account, fee_rate: int) -> bool: + coins = list(tx.reserved_utxos.order_by("id")) + if not coins or not tx.round.pool_address: + return False + owner = DepositAddress.objects.get(user=tx.user) + key = hd.user_key(account, owner.derivation_index) + bet = _bet_of(tx) + built = build_signed_transaction( + signing_key=key, from_script=script.p2wpkh(key.to_public()), + utxos=[EngineUtxo(c.txid, c.vout, c.amount_sats) for c in coins], + to_address=tx.round.pool_address, amount_sats=-bet.movements.get(kind=Movement.Kind.BET).amount_sats, + change_address=owner.address, fee_rate_sat_vb=fee_rate, + ) + with transaction.atomic(): + fresh = ChainTransaction.objects.select_for_update().get(pk=tx.pk) + if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid is not None: + return False # someone else got there first + register_version(fresh, built.txid, built.raw_hex) + bet.bet_txid, bet.bet_vout, bet.bet_amount_sats = built.txid, RECIPIENT_VOUT, built.recipient_sats + bet.save(update_fields=["bet_txid", "bet_vout", "bet_amount_sats"]) + audit.record("bet_signed", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, + payload={"txid": built.txid, "fee_sats": built.fee_sats, "recipient_sats": built.recipient_sats, + "inputs": len(coins)}) + return True + + +def _fail(tx: ChainTransaction, reason: str, *, release: bool) -> None: + """Refused with proof (or unbuildable): the bet is rejected, the coins return, the movement stops counting.""" + with transaction.atomic(): + fresh = ChainTransaction.objects.select_for_update().get(pk=tx.pk) + if fresh.status not in (ChainTransaction.Status.BUILDING, ChainTransaction.Status.PENDING): + return + fresh.status, fresh.failed_at, fresh.failure_reason = ChainTransaction.Status.FAILED, timezone.now(), reason[:500] + fresh.save(update_fields=["status", "failed_at", "failure_reason"]) + if release: + reservation.release(fresh) + bet = _bet_of(fresh) + if bet is not None: + bet.status = Participation.Status.REJECTED + bet.save(update_fields=["status"]) + Movement.objects.filter(participation=bet, kind=Movement.Kind.BET).update(status=Movement.Status.FAILED) + audit.record("bet_refused", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, + outcome="error", payload={"reason": reason[:500]}) + + +def _settle(tx_id: int, txid: str, verdict, now: datetime) -> bool: + """BET-33 / TX-26: write the outcome only if the row is as it was when the network was asked.""" + with transaction.atomic(): + fresh = ChainTransaction.objects.select_for_update().get(pk=tx_id) + if fresh.status != ChainTransaction.Status.BUILDING or fresh.current_txid != txid: + return False + if verdict.outcome is Outcome.ACCEPTED: + fresh.status, fresh.broadcast_at = ChainTransaction.Status.PENDING, now + fresh.save(update_fields=["status", "broadcast_at"]) + audit.record("bet_broadcast", actor_label=WORKER, subject_type="chain_transaction", subject_id=fresh.pk, + payload={"txid": txid, "note": verdict.reason}) + return True + if verdict.outcome is Outcome.REFUSED: + _fail(ChainTransaction.objects.get(pk=tx_id), verdict.reason or "refused by the network", release=True) + return True + return False # UNCERTAIN: nothing changes, the same transaction is sent again at the next pass + + +async def _send(chain, items: list[tuple[int, str, str]]): + return [(tx_id, txid, await chain.broadcast(raw, txid)) for tx_id, txid, raw in items] + + +def broadcast_pending(chain, now: datetime | None = None) -> dict[str, int]: + """BET-32: send every signed, unsent bet transaction and write what came of it. The database is not open while + the network is asked.""" + now = now or timezone.now() + done = {"accepted": 0, "refused": 0, "uncertain": 0} + waiting = ChainTransaction.objects.filter( + kind=ChainTransaction.Kind.BET, status=ChainTransaction.Status.BUILDING, current_txid__isnull=False, + ).order_by("id")[:_BATCH] + items = [(t.pk, t.current_txid, t.raw_hex) for t in waiting] + if not items: + return done + for tx_id, txid, verdict in asyncio.run(_send(chain, items)): + _settle(tx_id, txid, verdict, now) + done[verdict.outcome.value] += 1 + return done + + +def process_bets(chain, now: datetime | None = None) -> dict[str, int]: + signed = sign_pending() + done = broadcast_pending(chain, now) + return {"signed": signed, **done} diff --git a/apps/lottery/chain.py b/apps/lottery/chain.py index e123cc6..dc2e686 100644 --- a/apps/lottery/chain.py +++ b/apps/lottery/chain.py @@ -21,6 +21,11 @@ def derive_deposit_address(index: int) -> str: return hd.user_address(keys.public_account_key(), index, settings.PLM_PROFILE) +def derive_pool_address(round_id: int) -> str: + """The round's own pool address, `…/2/` under the account (DAT-60, RND-35): public key only.""" + return hd.pool_address(keys.public_account_key(), round_id, settings.PLM_PROFILE) + + def assign_deposit_address(user) -> DepositAddress: """The player's deposit address; created on first use, the same one ever after (DAT-81). The index is the next after the highest assigned; if a concurrent assignment takes it first the unique index refuses ours diff --git a/apps/lottery/deposits.py b/apps/lottery/deposits.py index 434075f..303e927 100644 --- a/apps/lottery/deposits.py +++ b/apps/lottery/deposits.py @@ -18,6 +18,7 @@ from .models import Deposit, DepositAddress, LotteryConfig, Movement, Transactio logger = logging.getLogger(__name__) +WAITING = ("detected", "orphaned") # deposit statuses that are still being watched COINBASE_MATURITY = 120 # DEP-21: PLM coinbase maturity (palladiumcore consensus.h) @@ -75,14 +76,14 @@ def plan() -> Policy: addresses=tuple(DepositAddress.objects.order_by("derivation_index").values_list("address", flat=True)), own_txids=frozenset(TransactionVersion.objects.values_list("txid", flat=True)), deposit_confirmations=LotteryConfig.get().deposit_confirmations, - credited=frozenset(Utxo.objects.filter(deposit__status=Deposit.Status.CREDITED).values_list("txid", "vout")), + credited=frozenset(Utxo.objects.exclude(deposit__status__in=WAITING).values_list("txid", "vout")), waiting=_waiting_by_address(), ) def _waiting_by_address() -> dict[str, frozenset[tuple[str, int]]]: grouped: dict[str, set[tuple[str, int]]] = {} - rows = Utxo.objects.filter(deposit__status__in=[Deposit.Status.DETECTED, Deposit.Status.ORPHANED]) + rows = Utxo.objects.filter(deposit__status__in=WAITING) for address, txid, vout in rows.values_list("address", "txid", "vout"): grouped.setdefault(address, set()).add((txid, vout)) return {address: frozenset(outpoints) for address, outpoints in grouped.items()} @@ -119,6 +120,13 @@ def apply_observations(result: ScanResult, policy: Policy, now: datetime | None def _apply_entry(owner, entry: Seen, tip: int, policy: Policy, observation: Observation, now, done) -> None: outpoint = (entry.txid, entry.vout) + if entry.txid in policy.own_txids: + # The change of one of our own transactions is spendable money, not a deposit: it was already taken + # from the player's balance when the transaction was placed (BET-30), so no movement is written. + with transaction.atomic(): + Utxo.objects.get_or_create(txid=entry.txid, vout=entry.vout, defaults=dict( + amount_sats=entry.value, address=owner.address, user=owner.user, confirmed_height=entry.height, created_at=now)) + return have = confirmations(tip, entry.height) need = threshold_for(entry.txid, policy, observation.coinbase) ready = have >= need and outpoint in observation.verified diff --git a/apps/lottery/electrum_chain.py b/apps/lottery/electrum_chain.py index b564d32..0bb634b 100644 --- a/apps/lottery/electrum_chain.py +++ b/apps/lottery/electrum_chain.py @@ -12,7 +12,10 @@ import logging from collections.abc import Awaitable, Callable from embit.transaction import Transaction -from plm_engine.electrum.client import ElectrumClient, ElectrumEndpoint +from enum import Enum +from typing import NamedTuple + +from plm_engine.electrum.client import ElectrumClient, ElectrumEndpoint, ElectrumRpcError from plm_engine.electrum.merkle import proof_matches_header from plm_engine.electrum.scripthash import address_to_scripthash from plm_engine.headers import header_meets_its_own_target @@ -27,6 +30,23 @@ CONNECT_TIMEOUT = 10.0 ANSWER_TIMEOUT = 10.0 +class Outcome(Enum): + ACCEPTED = "accepted" + REFUSED = "refused" + UNCERTAIN = "uncertain" + + +class Verdict(NamedTuple): + outcome: Outcome + reason: str + + +# Wordings by which a node says it already has the transaction. Recognising one skips the verification; not +# recognising one never releases anything, it only costs one more round of questions (BET-32, TX-04). +_ALREADY_KNOWN = ("already known", "already in mempool", "already in the mempool", "txn-already-known", + "txn-already-in-mempool", "already in block chain", "already in blockchain") + + def _seen(raw: dict) -> Seen | None: try: return Seen(txid=str(raw["tx_hash"]), vout=int(raw["tx_pos"]), value=int(raw["value"]), height=int(raw["height"])) @@ -177,6 +197,71 @@ class ElectrumChain: return self.majority(await self._ask_others(lacks_it)) + # -- broadcast (BET-32) ----------------------------------------------------------------------------------- + + async def transaction_known(self, active: ElectrumClient, txid: str) -> bool | None: + """True if any server returns the transaction; False only if the active server and the quorum of the + others all say they do not know it; None if there were not enough answers to say.""" + not_found = ("no such mempool", "no such blockchain", "not found", "unknown transaction") + + async def ask(client: ElectrumClient): + try: + await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT) + return True + except ElectrumRpcError as error: # the server answered: it does not know it + return False if any(m in error.message.lower() for m in not_found) else None + except Exception: + return None + + mine = await ask(active) + others = await self._ask_others(ask) + if mine is True or any(a is True for a in others): + return True + if mine is False and self.majority([a is False if a is not None else None for a in others]): + return False + return None + + async def broadcast(self, raw_tx_hex: str, txid: str) -> Verdict: + """Send `raw_tx_hex` through the active server and say what happened (BET-32). Only a proven refusal frees coins.""" + active = await self._connect_active() + if active is None: + return Verdict(Outcome.UNCERTAIN, "no Electrum server is reachable") + try: + try: + await asyncio.wait_for(active.broadcast(raw_tx_hex), ANSWER_TIMEOUT) + except ElectrumRpcError as error: + reason = error.message + known = True if any(m in reason.lower() for m in _ALREADY_KNOWN) else await self.transaction_known(active, txid) + if known is True: + await self._fan_out(raw_tx_hex) + return Verdict(Outcome.ACCEPTED, reason) + if known is False: + return Verdict(Outcome.REFUSED, reason) + logger.warning("broadcast of %s was refused (%s) but the refusal could not be verified", txid, reason) + return Verdict(Outcome.UNCERTAIN, reason) + except Exception as error: # timeout, dropped connection: the server may have relayed it + logger.warning("broadcast of %s ended without an answer (%r): uncertain", txid, error) + return Verdict(Outcome.UNCERTAIN, str(error) or type(error).__name__) + await self._fan_out(raw_tx_hex) + return Verdict(Outcome.ACCEPTED, "") + finally: + await active.close() + + async def _fan_out(self, raw_tx_hex: str) -> None: + """TX-21: after the active server accepted, tell the others too (best effort).""" + + async def send(client: ElectrumClient): + try: + await client.broadcast(raw_tx_hex) + except ElectrumRpcError: + pass # "already known" and the like: that is what we wanted + return True + + try: + await self._ask_others(send) + except Exception: + logger.exception("fan-out of an accepted transaction failed") + # -- the scan ---------------------------------------------------------------------------------------- async def observe(self, policy: Policy) -> ScanResult: diff --git a/apps/lottery/ledger.py b/apps/lottery/ledger.py index 929359f..3e03c97 100644 --- a/apps/lottery/ledger.py +++ b/apps/lottery/ledger.py @@ -67,6 +67,39 @@ def _simulated_txid(kind: str) -> str: return hashlib.sha256(f"simulated:{kind}:{uuid.uuid4()}".encode()).hexdigest() +def _admit_bet(ledger: "Ledger", user, round_id: int, request_id: UUID): + """The checks every bet passes, whatever the ledger (WEB-22, BET-07): a replay of the same request, an open + round, no suspension, no second bet, room in the round, and a balance that covers the quota. Returns a + `Result` for a replay, else `(round, amount)`. Call inside `transaction.atomic()`.""" + if not isinstance(request_id, UUID): + raise LedgerError("missing_request_id", "a bet needs the idempotency key of its request") + round_ = Round.objects.filter(pk=round_id).first() + replay = ledger._replay(user, request_id, Movement.Kind.BET, None) + if replay is not None: + if replay.round_id != round_id: + raise LedgerError("idempotency_conflict", "this request key was used for another round") + return Result(replay.participation, replayed=True) + if round_ is None or round_.status == Round.Status.CLOSED: + raise LedgerError("no_round_open", "no round open right now, please try again shortly") + if LotteryConfig.get().new_bets_suspended: + raise LedgerError("bets_suspended", "new bets are suspended") + if not round_accepts_bets(round_): + raise LedgerError("round_closing", "the current round is closing, please try again shortly") + if Participation.objects.filter(round=round_, user=user).exists(): + raise LedgerError("already_betting", "you already have an active bet in the current round") + SimulatedLedger._check_round_capacity(round_) + amount = round_.bet_amount_sats + balance = balance_of(user) + if balance.spendable_sats < amount: + # BET-07: a balance on screen that includes funds in confirmation must say "wait", not a flat "insufficient". + if balance.total_sats >= amount: + raise LedgerError("balance_pending_confirmation", + "the bet cost is covered by your pending balance, which has not confirmed yet", + pending_sats=balance.incoming_sats) + raise LedgerError("insufficient_balance", "insufficient balance", required_sats=amount) + return round_, amount + + class SimulatedLedger(Ledger): """Funds that exist only in the database. Confirmation is a method call, not a block.""" @@ -108,36 +141,11 @@ class SimulatedLedger(Ledger): def place_bet(self, user, round_id: int, *, request_id: UUID) -> Result: """WEB-22: the server decides — round, quota, limits and funds are all checked here.""" - if not isinstance(request_id, UUID): - raise LedgerError("missing_request_id", "a bet needs the idempotency key of its request") with transaction.atomic(): - round_ = Round.objects.filter(pk=round_id).first() - replay = self._replay(user, request_id, Movement.Kind.BET, None) - if replay is not None: - if replay.round_id != round_id: - raise LedgerError("idempotency_conflict", "this request key was used for another round") - return Result(replay.participation, replayed=True) - - if round_ is None or round_.status == Round.Status.CLOSED: - raise LedgerError("no_round_open", "no round open right now, please try again shortly") - if LotteryConfig.get().new_bets_suspended: - raise LedgerError("bets_suspended", "new bets are suspended") - if not round_accepts_bets(round_): - raise LedgerError("round_closing", "the current round is closing, please try again shortly") - if Participation.objects.filter(round=round_, user=user).exists(): - raise LedgerError("already_betting", "you already have an active bet in the current round") - - self._check_round_capacity(round_) - amount = round_.bet_amount_sats - balance = balance_of(user) - if balance.spendable_sats < amount: - # BET-07: a balance on screen that includes funds in confirmation must say "wait", - # not a flat "insufficient". - if balance.total_sats >= amount: - raise LedgerError("balance_pending_confirmation", - "the bet cost is covered by your pending balance, which has not confirmed yet", - pending_sats=balance.incoming_sats) - raise LedgerError("insufficient_balance", "insufficient balance", required_sats=amount) + admitted = _admit_bet(self, user, round_id, request_id) + if isinstance(admitted, Result): + return admitted + round_, amount = admitted txid = _simulated_txid("bet") chain_tx = ChainTransaction.objects.create( @@ -301,7 +309,62 @@ class SimulatedLedger(Ledger): return existing +class ChainLedger(SimulatedLedger): + """Real coins (roadmap B6). The web process admits and *reserves*; it never signs (ARC-78) — the worker does, + from `apps/lottery/bets.py`. What belongs to later steps refuses loudly instead of pretending: confirmation, + refusal and exclusion of a bet and the payout are B7/B8, withdrawals B9. Deposits arrive from the chain + (`deposits.scan_deposits`), never by a call.""" + + def place_bet(self, user, round_id: int, *, request_id: UUID) -> Result: + from . import reservation + from plm_engine.psbt_builder import InsufficientFundsError + + with transaction.atomic(): + admitted = _admit_bet(self, user, round_id, request_id) + if isinstance(admitted, Result): + return admitted + round_, amount = admitted + chain_tx = ChainTransaction.objects.create( + kind=ChainTransaction.Kind.BET, status=ChainTransaction.Status.BUILDING, user=user, round=round_ + ) + try: + coins = reservation.reserve_for(user, amount, chain_tx) + except InsufficientFundsError as error: + raise LedgerError(error.code, str(error), **error.params) from error + except reservation.CoinsTaken as error: + raise LedgerError("busy", "please try again in a moment") from error + participation = Participation.objects.create(round=round_, user=user, bet_amount_sats=amount) + Movement.objects.create( + user=user, kind=Movement.Kind.BET, amount_sats=-amount, status=Movement.Status.PENDING, + round=round_, participation=participation, chain_transaction=chain_tx, request_id=request_id, + ) + audit.record("bet_placed", actor=user, subject_type="participation", subject_id=participation.pk, + payload={"round_id": round_.pk, "amount_sats": amount, "coins": len(coins)}) + return Result(participation) + + def deposit(self, user, amount_sats: int, *, request_id: UUID | None = None, confirmed: bool = True) -> Result: + raise LedgerError("not_available", "deposits on a real ledger come from the chain, not from a call") + + def confirm_deposit(self, movement: Movement) -> Movement: + raise LedgerError("not_available", "deposits on a real ledger come from the chain, not from a call") + + def confirm_bet(self, participation: Participation) -> Participation: + raise NotImplementedError("the confirmation of a real bet arrives with roadmap B7") + + def reject_bet(self, participation: Participation, reason: str) -> Participation: + raise NotImplementedError("the refusal of a real bet after the fact arrives with roadmap B7") + + def exclude_bet(self, participation: Participation, reason: str) -> Participation: + raise NotImplementedError("the exclusion of a real bet arrives with roadmap B7") + + def pay_winner(self, round_id: int) -> Result: + raise NotImplementedError("the real payout arrives with roadmap B8") + + def confirm_payout(self, round_id: int) -> Round: + raise NotImplementedError("the real payout arrives with roadmap B8") + + def get_ledger() -> Ledger: if settings.LEDGER_BACKEND == "simulated": return SimulatedLedger() - raise NotImplementedError("the chain ledger arrives with phase B (roadmap B6)") + return ChainLedger() diff --git a/apps/lottery/lifecycle.py b/apps/lottery/lifecycle.py index 7a5447f..547181f 100644 --- a/apps/lottery/lifecycle.py +++ b/apps/lottery/lifecycle.py @@ -87,6 +87,11 @@ def open_new_round_if_needed(now: datetime | None = None) -> Round | None: server_seed_hash=server_seed_commitment(seed), # public from now on (RND-34) server_seed_enc=encrypt_seed(seed), # the secret itself stays hidden until the reveal ) + if settings.LEDGER_BACKEND == "chain": # DAT-60: this round's own address, from the public key + from .chain import derive_pool_address + + round_.pool_address = derive_pool_address(round_.pk) + round_.save(update_fields=["pool_address"]) except IntegrityError: # another process opened one first (ADR-22): use theirs return get_active_round() audit.record("round_opened", actor_label=WORKER, subject_type="round", subject_id=round_.pk, diff --git a/apps/lottery/management/commands/run_worker.py b/apps/lottery/management/commands/run_worker.py index 13c49a3..4b021ae 100644 --- a/apps/lottery/management/commands/run_worker.py +++ b/apps/lottery/management/commands/run_worker.py @@ -38,6 +38,19 @@ def scan_chain_deposits() -> None: logger.info("deposit scan: %s", done) +def process_chain_bets() -> None: + """BET-31/32: sign what the web admitted, send what was signed. On a real ledger only.""" + from django.conf import settings + + from apps.lottery import bets + from apps.lottery.electrum_chain import ElectrumChain + + chain = ElectrumChain(settings.ELECTRUM_ENDPOINTS, settings.PLM_PROFILE, settings.ELECTRUM_ALLOW_NO_QUORUM) + done = bets.process_bets(chain) + if any(done.values()): + logger.info("bets: %s", done) + + def tick() -> None: """One pass over everything the worker is responsible for.""" from django.conf import settings @@ -49,6 +62,7 @@ def tick() -> None: simulation.settle() else: scan_chain_deposits() # B5 + process_chain_bets() # B6 lifecycle.tick() diff --git a/apps/lottery/migrations/0007_fee_rate.py b/apps/lottery/migrations/0007_fee_rate.py new file mode 100644 index 0000000..57ada59 --- /dev/null +++ b/apps/lottery/migrations/0007_fee_rate.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.19 on 2026-11-16 15:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('lottery', '0006_deposit_confirmations'), + ] + + operations = [ + migrations.AddField( + model_name='lotteryconfig', + name='fee_rate_sat_vb', + field=models.PositiveIntegerField(default=1), + ), + migrations.AddConstraint( + model_name='lotteryconfig', + constraint=models.CheckConstraint(condition=models.Q(('fee_rate_sat_vb__gte', 1), ('fee_rate_sat_vb__lte', 5000)), name='lottery_config_fee_rate_range'), + ), + ] diff --git a/apps/lottery/models.py b/apps/lottery/models.py index 1f8b9c3..df6a6c0 100644 --- a/apps/lottery/models.py +++ b/apps/lottery/models.py @@ -35,6 +35,8 @@ class LotteryConfig(models.Model): # applies at once to the credits that follow. deposit_confirmations = models.PositiveSmallIntegerField(default=6) max_round_pool_sats = models.BigIntegerField(default=0) # 0 = no cap + # Network fee of the transactions built from now on, sat/vB (BET-04); a transaction in flight keeps its own. + fee_rate_sat_vb = models.PositiveIntegerField(default=1) # ADR-57: three distinct commands. pause_after_round = models.BooleanField(default=False) @@ -67,6 +69,9 @@ class LotteryConfig(models.Model): condition=Q(deposit_confirmations__gte=1, deposit_confirmations__lte=100), name="lottery_config_deposit_confirmations_range", ), + models.CheckConstraint( + condition=Q(fee_rate_sat_vb__gte=1, fee_rate_sat_vb__lte=5000), name="lottery_config_fee_rate_range" + ), ] @classmethod diff --git a/apps/lottery/reservation.py b/apps/lottery/reservation.py new file mode 100644 index 0000000..4efba6d --- /dev/null +++ b/apps/lottery/reservation.py @@ -0,0 +1,55 @@ +"""Which coins a player may spend, and taking them without anyone else taking them too (BET-30, ARC-72). + +A coin is spendable when it is confirmed, accredited (an own change output, or a deposit that reached its +threshold — never one still waiting, DEP-23), not spent and not reserved. Reserving is one conditional `UPDATE` +inside the caller's database transaction: it matches only coins whose `reserved_by` is still empty, and the +number of rows it touched must be the number asked for. Nothing in memory decides who gets a coin. +""" + +from django.db import transaction +from django.db.models import Q +from plm_engine.psbt_builder import DUST_LIMIT_SATS, InsufficientFundsError, select_utxos +from plm_engine.psbt_builder import Utxo as EngineUtxo + +from .models import ChainTransaction, Deposit, Utxo + + +class CoinsTaken(Exception): + """Some of the chosen coins were reserved by someone else between the choice and the reservation.""" + + +def spendable(user): + return Utxo.objects.filter( + Q(deposit__isnull=True) | Q(deposit__status=Deposit.Status.CREDITED), + user=user, confirmed_height__isnull=False, spent_txid__isnull=True, reserved_by__isnull=True, + ).order_by("-amount_sats", "id") + + +def reserve_for(user, amount_sats: int, tx: ChainTransaction) -> list[Utxo]: + """Choose coins covering `amount_sats` (fee comes out of the amount, change stays above dust) and reserve + them for `tx`. Raises `InsufficientFundsError` if they do not cover it, `CoinsTaken` if someone was faster.""" + with transaction.atomic(): + coins = list(spendable(user)) + selected, total = select_utxos( + [EngineUtxo(c.txid, c.vout, c.amount_sats) for c in coins], amount_sats, min_change_sats=DUST_LIMIT_SATS + ) + if total - amount_sats < DUST_LIMIT_SATS: + # B-62: the bet keeps a change output above dust so the transaction stays fee-bumpable; refused here, before + # anything is reserved, instead of when the worker comes to sign. + raise InsufficientFundsError( + f"the amount leaves no change output: {DUST_LIMIT_SATS - (total - amount_sats)} more sats are needed", + code="balance_leaves_no_change", required_extra_sats=DUST_LIMIT_SATS - (total - amount_sats)) + by_outpoint = {(c.txid, c.vout): c for c in coins} + chosen = [by_outpoint[(u.txid, u.vout)] for u in selected] + taken = Utxo.objects.filter(pk__in=[c.pk for c in chosen], reserved_by__isnull=True).update(reserved_by=tx) + if taken != len(chosen): + raise CoinsTaken(f"{len(chosen) - taken} of {len(chosen)} coins were taken") + return chosen + + +def release(tx: ChainTransaction) -> int: + """Give the coins of `tx` back. Only ever called on proof that the transaction never left (BET-32).""" + return Utxo.objects.filter(reserved_by=tx, spent_txid__isnull=True).update(reserved_by=None) + + +__all__ = ["CoinsTaken", "InsufficientFundsError", "release", "reserve_for", "spendable"] diff --git a/docker-compose.yml b/docker-compose.yml index b074e28..9fbc21a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,9 +54,13 @@ services: environment: - DJANGO_DB_PATH=/app/db_data/site.sqlite3 - PROCESS_ROLE=worker + # ARC-78: the signing key is mounted here and nowhere else (read-only); `manage.py init_keys` writes it. + - PLM_MASTER_KEY_FILE=/run/plm-keys/master.key + - PLM_KEY_ENCRYPTION_KEY_FILE=/run/plm-keys/master.encryption.key command: ["/bin/sh", "-c", "umask 077; exec python manage.py run_worker"] volumes: - ./data/db:/app/db_data + - ./data/keys:/run/plm-keys:ro depends_on: web: condition: service_healthy diff --git a/docs/architettura.md b/docs/architettura.md index 0be0099..5b740f4 100644 --- a/docs/architettura.md +++ b/docs/architettura.md @@ -986,7 +986,7 @@ non è rimosso le sezioni precedenti descrivono lo stato storico. Scelte: [ADR-5 cifrata e la chiave che la cifra, due file diversi) si carica solo se `PROCESS_ROLE=worker` e i file sono montati; in qualunque altro processo `signing_account_key()` fallisce con `SigningKeyUnavailable`, anche se i file ci fossero. `manage.py init_keys` crea le chiavi una tantum, scrive i file con modo `0600` e stampa la sola chiave pubblica. - *Stato:* implementato (B4); il montaggio dei file nel solo servizio `worker` arriva con B6, quando il worker firma · + *Stato:* implementato (B4); i file sono montati, in sola lettura, nel solo servizio `worker` del compose (`./data/keys`, B6) · *Codice:* `apps/lottery/keys.py`, `apps/lottery/management/commands/init_keys.py` · *Test:* `tests/django/test_chain_models.py::test_the_web_process_cannot_load_the_signing_key`, `test_the_worker_can_and_the_files_are_private` diff --git a/docs/legacy-fastapi-notes.md b/docs/legacy-fastapi-notes.md index 133db02..f6dfec6 100644 --- a/docs/legacy-fastapi-notes.md +++ b/docs/legacy-fastapi-notes.md @@ -4,7 +4,7 @@ ## Project status (old stack) -All 10 stages of the original build order are code-complete and unit-tested — 1263 tests, the old stack's under `tests/unit/` and the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Beyond them: Docker + Caddy deployment, admin dashboard (`/admin`), static test UI (`/`), pending-inclusive balance display, an SSE push channel layered over the original polling, self-service password change + admin password reset, and the reconciliation/corroboration machinery below. +All 10 stages of the original build order are code-complete and unit-tested — 1288 tests, the old stack's under `tests/unit/` and the Django ones under `tests/django/` (`tests/integration/` needs the local regtest node and skips itself without it). Beyond them: Docker + Caddy deployment, admin dashboard (`/admin`), static test UI (`/`), pending-inclusive balance display, an SSE push channel layered over the original polling, self-service password change + admin password reset, and the reconciliation/corroboration machinery below. Verified on mainnet with real money: registration + address derivation, deposit crediting (at the time, 1-conf; deposits now need `deposit_confirmations`), a real 10 PLM bet (broadcast → confirmed → change credited back), and one full round cycle (close → draw on a real block hash → 70/30 payout with sat math checked against the broadcast tx → confirmation → close → next round auto-opened). **Withdrawal and the RBF bump path have never been exercised against a live broadcast** — unit-tested only. @@ -30,7 +30,7 @@ PYTHONPATH=. python scripts/decrypt_master_key.py # ops recovery: decrypt+pr PYTHONPATH=. python scripts/encrypt_master_key.py # ops bootstrap: import an externally-generated xprv (--overwrite to replace) PYTHONPATH=. python scripts/electrum_smoke_test.py # manual check: connect, handshake, subscribe to headers, print the tip -python -m pytest # all 1263 tests +python -m pytest # all 1288 tests python -m pytest tests/unit/test_hd.py # one file python -m pytest tests/unit/test_hd.py::test_derivation_is_deterministic # one test ``` diff --git a/docs/modello-dati.md b/docs/modello-dati.md index 7dcaad4..6dfa4d6 100644 --- a/docs/modello-dati.md +++ b/docs/modello-dati.md @@ -1500,7 +1500,7 @@ Le tabelle di catena (UTXO, versioni delle transazioni, depositi, indici HD) son si elimina. Ogni importo è un intero in unità minime ([DAT-01]) e ogni timestamp è in UTC con fuso ([DAT-02]). *Stato:* implementato (A2) · *Codice:* `apps/lottery/models.py` · *Test:* `tests/django/test_models.py::test_rows_that_carry_money_cannot_be_deleted_with_their_owner` - **[DAT-77]** **Parametri e pause: `LotteryConfig`.** Una sola riga (`id = 1`, vincolo) con quota (10 PLM), - durata (600 s), pausa fra round (30 s), conferme richieste a un deposito (1–100, default 6, [DAT-62]; si cambia da `/admin/` con audit e vale subito per gli accrediti successivi), conferme del blocco di estrazione (1–20, default 3), tetto del + durata (600 s), pausa fra round (30 s), conferme richieste a un deposito (1–100, default 6, [DAT-62]; si cambia da `/admin/` con audit e vale subito per gli accrediti successivi), conferme del blocco di estrazione (1–20, default 3), commissione di rete in sat/vB (1–5 000, default 1: vale per le transazioni costruite dopo), tetto del montepremi (0 = nessuno) e i tre interruttori di [ADR-57](adr/ADR-57-due-pause-distinte-dopo-round-e-incidente.md): `pause_after_round`, `new_bets_suspended`, `withdrawals_suspended`. Valori non positivi o fuori intervallo sono rifiutati dal database. I parametri si **copiano su `Round` all'apertura** ([DAT-61]); la copia la fa il diff --git a/docs/scommesse-e-prelievi.md b/docs/scommesse-e-prelievi.md index 198e03b..ab65b28 100644 --- a/docs/scommesse-e-prelievi.md +++ b/docs/scommesse-e-prelievi.md @@ -349,3 +349,39 @@ un solo processo (vedi [architettura.md](architettura.md) e Cosa succede a una transazione dopo l'invio — conferma, RBF, abbandono — è descritto in [transazioni-e-riconciliazione.md](transazioni-e-riconciliazione.md). + +## Scommessa reale sul sito Django (B6) + +Con `LEDGER_BACKEND=chain` la scommessa muove monete vere. Le regole di ammissione ([BET-01]…[BET-23]: round, quota, limiti, +una scommessa per giocatore e round, saldo in conferma) sono le stesse del ledger simulato. Cambia ciò che segue l'ammissione. + +- **[BET-30]** **Prenotazione atomica degli input.** L'ammissione sceglie le monete del giocatore (confermate, accreditate, + non spese e **non prenotate**: i depositi ancora in attesa non contano, [DEP-23]) e le **prenota** nella stessa transazione del + database che scrive la scommessa: `Utxo.reserved_by` punta alla transazione della piattaforma, con un `UPDATE` condizionato a + `reserved_by` vuoto e a un numero di righe uguale a quello scelto. Due richieste concorrenti non possono quindi spendere la + stessa moneta, qualunque sia l'ordine: una prenota, l'altra non trova monete libere. Una moneta prenotata non è mai offerta + a un'altra operazione. L'importo che il saldo derivato ([DAT-80]) toglie al giocatore è la quota, subito. + *Stato:* implementato (B6) · *Codice:* `apps/lottery/reservation.py`, `apps/lottery/ledger.py:ChainLedger.place_bet` · + *Test:* `tests/django/test_chain_bets.py::test_two_requests_never_reserve_the_same_coin`, `test_a_reserved_coin_is_not_offered_again`, `test_waiting_deposits_are_not_spendable` +- **[BET-31]** **Il web non firma; il worker salva prima di trasmettere.** La richiesta web lascia una `ChainTransaction` `building` + senza txid, con le monete prenotate e una `Participation` `pending`. Il **worker** (il solo con la chiave, [ARC-78]) costruisce e firma la + transazione e, **in una transazione del database**, registra la versione col txid e la transazione grezza ([DAT-83]) e aggiorna + la scommessa con l'importo effettivo al montepremi (quota meno commissione di rete, [BET-04]) e l'outpoint ([DAT-63]); **solo dopo** + trasmette. Un arresto fra i due passi lascia la transazione firmata e salvata: al giro dopo si ritrasmette la stessa. + Senza la chiave (processo non worker, file assenti) il worker non firma nulla e lo dice nei log. + *Stato:* implementato (B6) · *Codice:* `apps/lottery/bets.py:sign_pending`, `apps/lottery/keys.py` · + *Test:* `tests/django/test_chain_bets.py::test_the_transaction_is_saved_before_it_is_broadcast`, `test_without_the_signing_key_nothing_is_signed`, `test_the_stake_in_the_pool_is_the_quota_minus_the_fee` +- **[BET-32]** **Verdetto della trasmissione** ([TX-04], [TX-20], [TX-21]). Una trasmissione finisce in uno di tre modi: **accettata** + (il server risponde col txid, o rifiuta ma almeno un server conosce la transazione, anche con «già nota»), **rifiutata** + (rifiuto esplicito **e** il server attivo più il quorum minimo degli altri dicono di non conoscerla) o **incerta** (qualunque altra + cosa: timeout, connessione caduta, rifiuto non verificabile). Solo la rifiutata libera le monete: la scommessa passa a `rejected`, il movimento + a `failed`, le monete tornano libere, la transazione è `failed` con il motivo. L'accettata porta la transazione a `pending` e la + trasmette anche agli altri server. L'incerta non cambia nulla: resta `building` e al giro dopo si ritrasmette la stessa transazione, che un server + che l'ha già vista accetta come «già nota». + *Stato:* implementato (B6); conferma, RBF e abbandono con prova sono B7 · *Codice:* `apps/lottery/electrum_chain.py:ElectrumChain.broadcast`, `apps/lottery/bets.py:broadcast_pending` · + *Test:* `tests/django/test_chain_bets.py::test_an_accepted_broadcast_makes_the_bet_pending`, `test_a_proven_refusal_frees_the_coins`, `test_an_uncertain_broadcast_frees_nothing_and_is_retried`, `test_already_known_counts_as_accepted` +- **[BET-33]** **Esito scritto solo se lo stato non è cambiato** ([TX-26]). Prima di scrivere l'esito di una trasmissione il worker rilegge la + transazione nella propria transazione del database e non fa nulla se lo stato o il `current_txid` non sono più quelli con cui ha + interrogato la rete. + *Stato:* implementato (B6) · *Codice:* `apps/lottery/bets.py:_settle` · *Test:* `tests/django/test_chain_bets.py::test_an_outcome_is_not_written_over_a_changed_row` + diff --git a/roadmap.md b/roadmap.md index d48ea40..c71191f 100644 --- a/roadmap.md +++ b/roadmap.md @@ -44,7 +44,7 @@ Uscita: flusso completo su regtest; un riavvio durante il broadcast, due richies - [x] **B3 · Ambiente regtest e fixture.** Il nodo (`palladiumcore` in `-regtest`) e Fulcrum sono quelli della cartella `plm-regtest` accanto al repository (`docker compose up -d` lì): non si duplicano nel compose del sito, ci si aggancia. `plm_engine/regtest.py` (client RPC con rifiuto di toccare una catena diversa da regtest), `scripts/regtest.py` (`status`, `mine`, `fund`) e le fixture di `tests/integration/`, che si saltano se il nodo o Fulcrum non rispondono. *Test:* `tests/integration/test_regtest_smoke.py` (blocco minato, indirizzi e genesis del profilo, tip di Fulcrum, fondi visti via Electrum). - [x] **B4 · Modelli di catena.** UTXO, transazioni con tutte le versioni TXID, depositi, assegnazione dell'indice HD; la chiave di firma è accessibile solo al processo worker. *Requisiti:* DAT, TX. *Test:* vincoli, derivazione identica a parità di input. - [x] **B5 · Listener e depositi reali.** Listener Electrum sull'ORM con rotazione, header validati, quorum e soglie di conferma al posto del deposito simulato. *Requisiti:* DEP, ELC. *Test:* unit con server finto + integrazione su regtest. -- [ ] **B6 · Prenotazione UTXO e puntata reale.** Prenotazione atomica degli UTXO nel database, identificativo idempotente, «salva operazione e transazione firmata prima del broadcast», verdetto del broadcast (accettato, rifiutato, incerto). *Requisiti:* BET, TX-26. *Test:* due richieste concorrenti non spendono gli stessi input; timeout ambiguo riconciliato. +- [x] **B6 · Prenotazione UTXO e puntata reale.** Prenotazione atomica degli UTXO nel database, identificativo idempotente, «salva operazione e transazione firmata prima del broadcast», verdetto del broadcast (accettato, rifiutato, incerto). *Requisiti:* BET, TX-26. *Test:* due richieste concorrenti non spendono gli stessi input; timeout ambiguo riconciliato. - [ ] **B7 · Conferme, riconciliazione e RBF.** Poller, riconciliatore e bumper sull'ORM con tutte le versioni TXID e adozione della versione confermata. *Requisiti:* TX. *Test:* conferma di una versione RBF precedente, ripresa dopo riavvio. - [ ] **B8 · Estrazione e pagamento reali.** Commit-reveal con blocco a quorum, verifica del montepremi, pagamento 70/30 con riserva per l'RBF, pausa automatica sui casi bloccanti. *Requisiti:* RND. *Test:* round completo su regtest, riorganizzazione simulata. - [ ] **B9 · Prelievi e storico.** Prelievo con indirizzo, importo, commissione e netto previsti; storico con stato dell'operazione e TXID corrente con link all'explorer. *Requisiti:* WDR, WEB-24. *Test:* prelievo su regtest, sospensione dei prelievi. diff --git a/tests/django/test_chain_bets.py b/tests/django/test_chain_bets.py new file mode 100644 index 0000000..106705a --- /dev/null +++ b/tests/django/test_chain_bets.py @@ -0,0 +1,375 @@ +"""B6 — a real bet: reservation, signing in the worker, broadcast verdicts: BET-30..BET-33, TX-20, TX-26, ARC-72, ARC-78.""" + +import json +import os +import subprocess +import sys +from datetime import timedelta +from pathlib import Path +from uuid import uuid4 + +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone +from embit.transaction import Transaction +from plm_engine.electrum.client import ElectrumEndpoint, ElectrumRpcError +from plm_engine.psbt_builder import DUST_LIMIT_SATS + +from apps.lottery import bets, chain, keys, reservation +from apps.lottery.balances import balance_of +from apps.lottery.electrum_chain import ElectrumChain, Outcome, Verdict +from apps.lottery.errors import LedgerError +from apps.lottery.ledger import ChainLedger, get_ledger +from apps.lottery.models import ( + AuditEntry, ChainTransaction, Deposit, Movement, Participation, Round, Utxo, +) + +ROOT = Path(__file__).resolve().parents[2] +User = get_user_model() +PLM = 100_000_000 +BET = 10 * PLM +pytestmark = pytest.mark.django_db +ledger = ChainLedger() + + +@pytest.fixture +def world(settings, tmp_path): + """A real master key: the public part configured as the web has it, the private part as the worker has it.""" + settings.LEDGER_BACKEND = "chain" + settings.PLM_ACCOUNT_XPUB = keys.generate_keys(tmp_path / "keys", settings.PLM_PROFILE) + settings.PLM_MASTER_KEY_FILE = str(tmp_path / "keys" / "master.key") + settings.PLM_KEY_ENCRYPTION_KEY_FILE = str(tmp_path / "keys" / "master.encryption.key") + settings.PROCESS_ROLE = "worker" + return settings + + +def make_round(): + round_ = Round.objects.create(opened_at=timezone.now() - timedelta(seconds=30), bet_amount_sats=BET) + round_.pool_address = chain.derive_pool_address(round_.pk) + round_.save(update_fields=["pool_address"]) + return round_ + + +_counter = iter(range(1, 10_000)) + + +def fund(user, amount, *, credited=True): + """A deposit the scan has already registered: a coin of the player's own address.""" + address = chain.assign_deposit_address(user).address + n = next(_counter) + utxo = Utxo.objects.create(txid=f"{n:064x}", vout=0, amount_sats=amount, address=address, user=user, confirmed_height=100) + movement = Movement.objects.create( + user=user, kind="deposit", status="confirmed" if credited else "pending", amount_sats=amount, + confirmed_at=timezone.now() if credited else None) + Deposit.objects.create(user=user, utxo=utxo, movement=movement, status="credited" if credited else "detected", + credited_at=timezone.now() if credited else None) + return utxo + + +def player(name="mario", funds=3 * BET): + user = User.objects.create_user(name, password="x" * 12) + if funds: + fund(user, funds) + return user + + +def bet(user, round_): + return ledger.place_bet(user, round_.pk, request_id=uuid4()) + + +# --- BET-30: admission and reservation ---------------------------------------------------------------------- + + +def test_a_bet_reserves_its_coins_and_takes_the_quota_from_the_balance(world): + user, round_ = player(), make_round() + result = bet(user, round_) + tx = ChainTransaction.objects.get() + assert (tx.kind, tx.status, tx.current_txid) == ("bet", "building", None) + assert list(tx.reserved_utxos.values_list("amount_sats", flat=True)) == [3 * BET] + assert result.obj.status == "pending" and balance_of(user).spendable_sats == 2 * BET + + +def test_a_reserved_coin_is_not_offered_again(world): + user, round_ = player(funds=BET + 2 * PLM), make_round() + bet(user, round_) + assert not reservation.spendable(user).exists() + + +def test_waiting_deposits_are_not_spendable(world): + user, round_ = player(funds=0), make_round() + fund(user, 2 * BET, credited=False) + with pytest.raises(LedgerError) as refused: + bet(user, round_) + assert refused.value.code == "balance_pending_confirmation" and not ChainTransaction.objects.exists() + + +def test_a_balance_that_leaves_no_change_is_refused_with_the_documented_code(world): + user, round_ = player(funds=BET), make_round() + with pytest.raises(LedgerError) as refused: + bet(user, round_) + assert refused.value.code == "balance_leaves_no_change" + assert not ChainTransaction.objects.exists() and not Utxo.objects.filter(reserved_by__isnull=False).exists() + + +def test_not_enough_money_is_refused(world): + user, round_ = player(funds=PLM), make_round() + with pytest.raises(LedgerError) as refused: + bet(user, round_) + assert refused.value.code == "insufficient_balance" + + +def test_the_same_request_is_one_bet(world): + user, round_ = player(), make_round() + key = uuid4() + first = ledger.place_bet(user, round_.pk, request_id=key) + again = ledger.place_bet(user, round_.pk, request_id=key) + assert again.replayed and again.obj == first.obj and ChainTransaction.objects.count() == 1 + + +def test_a_second_bet_in_the_round_is_refused_and_reserves_nothing(world): + user, round_ = player(), make_round() + bet(user, round_) + with pytest.raises(LedgerError) as refused: + bet(user, round_) + assert refused.value.code == "already_betting" and ChainTransaction.objects.count() == 1 + + +def test_the_chain_ledger_does_not_pretend_what_is_not_there_yet(world): + with pytest.raises(LedgerError): + ledger.deposit(player(), PLM) + with pytest.raises(NotImplementedError): + ledger.pay_winner(1) + assert isinstance(get_ledger(), ChainLedger) + + +_SETUP = """ +import django, json +django.setup() +from uuid import uuid4 +from django.contrib.auth import get_user_model +from django.utils import timezone +from apps.lottery import chain +from apps.lottery.models import Utxo, Deposit, Movement +user = get_user_model().objects.create_user('mario', password='x'*12) +address = chain.assign_deposit_address(user).address +utxo = Utxo.objects.create(txid='ab'*32, vout=0, amount_sats=3_000_000_000, address=address, user=user, confirmed_height=1) +m = Movement.objects.create(user=user, kind='deposit', status='confirmed', amount_sats=3_000_000_000, confirmed_at=timezone.now()) +Deposit.objects.create(user=user, utxo=utxo, movement=m, status='credited', credited_at=timezone.now()) +""" + +_RACE = """ +import django, json +django.setup() +from django.contrib.auth import get_user_model +from apps.lottery import reservation +from apps.lottery.models import ChainTransaction +user = get_user_model().objects.get(username='mario') +tx = ChainTransaction.objects.create(kind='bet', user=user) +try: + reservation.reserve_for(user, 1_000_000_000, tx) + print(json.dumps('won')) +except Exception as exc: + print(json.dumps(type(exc).__name__)) +""" + + +def test_two_requests_never_reserve_the_same_coin(tmp_path, settings): + """ARC-72: six processes want the one coin of a player at once; exactly one gets it.""" + env = {"PATH": os.environ["PATH"], "PYTHONPATH": str(ROOT), "DJANGO_SECRET_KEY": "k" * 60, + "DJANGO_DB_PATH": str(tmp_path / "site.sqlite3"), "DJANGO_SETTINGS_MODULE": "config.settings", + "PLM_ACCOUNT_XPUB": keys.generate_keys(tmp_path / "keys", settings.PLM_PROFILE)} + subprocess.run([sys.executable, "manage.py", "migrate", "--noinput", "-v", "0"], cwd=ROOT, env=env, check=True) + subprocess.run([sys.executable, "-c", _SETUP], cwd=ROOT, env=env, check=True) + procs = [subprocess.Popen([sys.executable, "-c", _RACE], cwd=ROOT, env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) for _ in range(6)] + outputs = [p.communicate(timeout=120) for p in procs] + assert all(p.returncode == 0 for p in procs), [o[1] for o in outputs] + results = [json.loads(o[0].strip().splitlines()[-1]) for o in outputs] + assert results.count("won") == 1 and results.count("InsufficientFundsError") == 5, results + + +# --- BET-31: signing in the worker, saved before it is sent --------------------------------------------------- + + +def test_without_the_signing_key_nothing_is_signed(world): + user, round_ = player(), make_round() + bet(user, round_) + world.PROCESS_ROLE = "web" + assert bets.sign_pending() == 0 + tx = ChainTransaction.objects.get() + assert tx.current_txid is None and not tx.versions.exists() + + +def test_the_stake_in_the_pool_is_the_quota_minus_the_fee(world): + user, round_ = player(), make_round() + bet(user, round_) + assert bets.sign_pending() == 1 + tx = ChainTransaction.objects.get() + participation = Participation.objects.get() + parsed = Transaction.parse(bytes.fromhex(tx.raw_hex)) + assert tx.current_txid == parsed.txid().hex() and tx.versions.get().txid == tx.current_txid + assert participation.bet_txid == tx.current_txid and participation.bet_vout == 0 + pool_out, change_out = parsed.vout + assert pool_out.value == participation.bet_amount_sats < BET # the fee comes out of the stake (BET-04) + assert pool_out.value + (BET - pool_out.value) == BET + assert change_out.value == 3 * BET - BET and change_out.value >= DUST_LIMIT_SATS + assert pool_out.script_pubkey.address(world.PLM_PROFILE.embit) == round_.pool_address + assert [(i.txid.hex(), i.vout) for i in parsed.vin] == [(u.txid, u.vout) for u in tx.reserved_utxos.all()] + assert AuditEntry.objects.filter(action="bet_signed").count() == 1 + + +def test_a_signed_transaction_is_not_signed_twice(world): + user, round_ = player(), make_round() + bet(user, round_) + assert bets.sign_pending() == 1 and bets.sign_pending() == 0 + assert ChainTransaction.objects.get().versions.count() == 1 + + +# --- BET-32: verdicts ------------------------------------------------------------------------------------------ + + +class ScriptedChain: + """What `broadcast_pending` asks of the network: a verdict per call.""" + + def __init__(self, *verdicts): + self.verdicts, self.sent = list(verdicts), [] + + async def broadcast(self, raw, txid): + self.sent.append(txid) + return self.verdicts.pop(0) if self.verdicts else Verdict(Outcome.ACCEPTED, "") + + +def placed_and_signed(world): + user, round_ = player(), make_round() + bet(user, round_) + bets.sign_pending() + return user, round_, ChainTransaction.objects.get() + + +def test_the_transaction_is_saved_before_it_is_broadcast(world): + user, round_, tx = placed_and_signed(world) + assert tx.status == "building" and tx.current_txid and tx.versions.exists() # nothing was sent yet + class Crashing: + async def broadcast(self, raw, txid): + raise RuntimeError("the process died here") + with pytest.raises(RuntimeError): + bets.broadcast_pending(Crashing()) + tx.refresh_from_db() + assert tx.status == "building" and tx.versions.count() == 1 # saved, still unsent: the next pass sends it + + +def test_an_accepted_broadcast_makes_the_bet_pending(world): + user, round_, tx = placed_and_signed(world) + assert bets.broadcast_pending(ScriptedChain())["accepted"] == 1 + tx.refresh_from_db() + assert tx.status == "pending" and tx.broadcast_at is not None + assert AuditEntry.objects.filter(action="bet_broadcast").exists() + assert bets.broadcast_pending(ScriptedChain()) == {"accepted": 0, "refused": 0, "uncertain": 0} # not sent again + + +def test_a_proven_refusal_frees_the_coins(world): + user, round_, tx = placed_and_signed(world) + assert bets.broadcast_pending(ScriptedChain(Verdict(Outcome.REFUSED, "bad-txns-inputs-missingorspent")))["refused"] == 1 + tx.refresh_from_db() + assert tx.status == "failed" and "missingorspent" in tx.failure_reason and tx.failed_at + assert Participation.objects.get().status == "rejected" + assert Movement.objects.get(kind="bet").status == "failed" + assert reservation.spendable(user).count() == 1 and balance_of(user).spendable_sats == 3 * BET + assert AuditEntry.objects.filter(action="bet_refused").exists() + + +def test_an_uncertain_broadcast_frees_nothing_and_is_retried(world): + user, round_, tx = placed_and_signed(world) + assert bets.broadcast_pending(ScriptedChain(Verdict(Outcome.UNCERTAIN, "timeout")))["uncertain"] == 1 + tx.refresh_from_db() + assert tx.status == "building" and tx.reserved_utxos.count() == 1 and not reservation.spendable(user).exists() + retry = ScriptedChain() + assert bets.broadcast_pending(retry)["accepted"] == 1 and retry.sent == [tx.current_txid] # the same transaction + tx.refresh_from_db() + assert tx.status == "pending" and tx.versions.count() == 1 + + +def test_an_outcome_is_not_written_over_a_changed_row(world): + user, round_, tx = placed_and_signed(world) + other = "ee" * 32 + assert bets._settle(tx.pk, other, Verdict(Outcome.ACCEPTED, ""), timezone.now()) is False # another txid (an RBF meanwhile) + ChainTransaction.objects.filter(pk=tx.pk).update(status="pending") + assert bets._settle(tx.pk, tx.current_txid, Verdict(Outcome.REFUSED, "late"), timezone.now()) is False + tx.refresh_from_db() + assert tx.status == "pending" and tx.reserved_utxos.count() == 1 + + +# --- the verdict as the network phase reaches it --------------------------------------------------------------- + + +class Server: + def __init__(self, name, *, broadcast=None, knows=None): + self.name, self.broadcast_error, self.knows = name, broadcast, knows # knows: True / False / None (silent) + self.sent = [] + + +class FakeClient: + def __init__(self, server): + self.s = server + + async def connect(self): + pass + + async def close(self): + pass + + async def broadcast(self, raw): + self.s.sent.append(raw) + if self.s.broadcast_error is not None: + raise self.s.broadcast_error + return "txid" + + async def get_transaction(self, txid, verbose=False): + if self.s.knows is True: + return "00" + if self.s.knows is False: + raise ElectrumRpcError({"code": 2, "message": "No such mempool or blockchain transaction"}) + raise OSError("timeout") + + +def network(*servers, allow_no_quorum=False): + by_endpoint = {ElectrumEndpoint(s.name, 1, False): s for s in servers} + from plm_engine.network import REGTEST + + return ElectrumChain(list(by_endpoint), REGTEST, allow_no_quorum, lambda e: FakeClient(by_endpoint[e])) + + +async def test_a_plain_acceptance_is_fanned_out_to_the_others(): + servers = [Server("a"), Server("b"), Server("c")] + verdict = await network(*servers).broadcast("00ff", "tx") + assert verdict.outcome is Outcome.ACCEPTED and [len(s.sent) for s in servers] == [1, 1, 1] + + +async def test_already_known_counts_as_accepted(): + refusal = ElectrumRpcError({"code": -26, "message": "txn-already-in-mempool"}) + verdict = await network(Server("a", broadcast=refusal), Server("b"), Server("c")).broadcast("00ff", "tx") + assert verdict.outcome is Outcome.ACCEPTED + + +async def test_a_refusal_that_another_server_contradicts_is_an_acceptance(): + refusal = ElectrumRpcError({"code": 1, "message": "weird wording"}) + servers = [Server("a", broadcast=refusal, knows=False), Server("b", knows=True), Server("c", knows=False)] + assert (await network(*servers).broadcast("00ff", "tx")).outcome is Outcome.ACCEPTED + + +async def test_a_refusal_every_server_confirms_is_a_refusal(): + refusal = ElectrumRpcError({"code": 1, "message": "bad-txns-inputs-missingorspent"}) + servers = [Server("a", broadcast=refusal, knows=False), Server("b", knows=False), Server("c", knows=False)] + verdict = await network(*servers).broadcast("00ff", "tx") + assert verdict.outcome is Outcome.REFUSED and "missingorspent" in verdict.reason + + +async def test_a_refusal_the_others_cannot_confirm_is_only_uncertain(): + refusal = ElectrumRpcError({"code": 1, "message": "bad-txns-inputs-missingorspent"}) + servers = [Server("a", broadcast=refusal, knows=False), Server("b", knows=None), Server("c", knows=None)] + assert (await network(*servers).broadcast("00ff", "tx")).outcome is Outcome.UNCERTAIN + + +async def test_a_timeout_is_uncertain_and_never_a_refusal(): + servers = [Server("a", broadcast=OSError("timeout")), Server("b"), Server("c")] + assert (await network(*servers).broadcast("00ff", "tx")).outcome is Outcome.UNCERTAIN diff --git a/tests/django/test_deposits.py b/tests/django/test_deposits.py index 6c22501..00c9e66 100644 --- a/tests/django/test_deposits.py +++ b/tests/django/test_deposits.py @@ -89,14 +89,19 @@ def test_a_coinbase_needs_120(player): assert scan(fake)["credited"] == 1 -def test_our_own_transactions_need_one(player): +def test_the_change_of_our_own_transactions_is_spendable_but_never_a_deposit(player): + """BET-30: the stake was taken from the balance when the bet was placed; its change coming back must not be credited.""" user, address = player - tx = ChainTransaction.objects.create(kind="withdrawal", user=user) + tx = ChainTransaction.objects.create(kind="bet", user=user) chain.register_version(tx, TXID, "00") fake = FakeChain() fake.entries[address] = [coin(height=1000)] # one confirmation - fake.verified = {(TXID, 0)} - assert scan(fake)["credited"] == 1 + assert scan(fake) == {"detected": 0, "credited": 0, "orphaned": 0, "reinstated": 0} + utxo = Utxo.objects.get() + assert utxo.user == user and utxo.confirmed_height == 1000 and not Deposit.objects.exists() + assert not Movement.objects.exists() and balance_of(user).spendable_sats == 0 + scan(fake) # and once only + assert Utxo.objects.count() == 1 def test_dust_and_mempool_coins_are_ignored(player): @@ -217,7 +222,7 @@ def test_the_operator_changes_the_threshold_from_the_admin_and_it_is_audited(cli client.force_login(admin) config = LotteryConfig.get() data = {f: getattr(config, f) for f in ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds", - "draw_confirmations", "deposit_confirmations", "max_round_pool_sats")} + "draw_confirmations", "deposit_confirmations", "max_round_pool_sats", "fee_rate_sat_vb")} data["deposit_confirmations"] = 3 assert client.post("/admin/lottery/lotteryconfig/1/change/", data).status_code == 302 assert LotteryConfig.get().deposit_confirmations == 3 diff --git a/tests/django/test_ledger.py b/tests/django/test_ledger.py index 0eb4805..13fd19c 100644 --- a/tests/django/test_ledger.py +++ b/tests/django/test_ledger.py @@ -420,8 +420,9 @@ def test_unknown_network_or_ledger_is_refused(env): def test_get_ledger_follows_the_configuration(settings): assert isinstance(get_ledger(), SimulatedLedger) settings.LEDGER_BACKEND = "chain" - with pytest.raises(NotImplementedError): - get_ledger() + from apps.lottery.ledger import ChainLedger + + assert isinstance(get_ledger(), ChainLedger) # --- real concurrency: separate processes on one SQLite file ----------------------------------- diff --git a/tests/django/test_lifecycle.py b/tests/django/test_lifecycle.py index ef22075..0fcf142 100644 --- a/tests/django/test_lifecycle.py +++ b/tests/django/test_lifecycle.py @@ -104,15 +104,20 @@ def test_a_round_copies_its_rules_when_it_opens(t0): assert (round_.draw_confirmations, round_.cooldown_seconds, round_.fee_address) == (5, 11, "rplm1fee") -def test_a_real_ledger_needs_a_fee_address_to_open_a_round(settings, t0): - """ADR-29: a round that cannot pay its commission is not opened. The simulated ledger needs none.""" +def test_a_real_ledger_needs_a_fee_address_to_open_a_round(settings, t0, tmp_path): + """ADR-29: a round that cannot pay its commission is not opened. The simulated ledger needs none. A real one + also derives the round's pool address from the public key (DAT-60), so it needs that configured.""" + from apps.lottery import keys + assert lifecycle.rounds_can_open(LotteryConfig.get()) settings.LEDGER_BACKEND = "chain" + settings.PLM_ACCOUNT_XPUB = keys.generate_keys(tmp_path / "keys", settings.PLM_PROFILE) assert lifecycle.open_new_round_if_needed(t0) is None config = LotteryConfig.get() config.fee_address = "rplm1fee" config.save() - assert lifecycle.open_new_round_if_needed(t0) is not None + opened = lifecycle.open_new_round_if_needed(t0) + assert opened is not None and opened.pool_address.startswith("rplm1") # its own address, from the public key def test_the_pause_after_the_round_stops_only_the_next_one(t0): diff --git a/tests/integration/test_bet_regtest.py b/tests/integration/test_bet_regtest.py new file mode 100644 index 0000000..7629084 --- /dev/null +++ b/tests/integration/test_bet_regtest.py @@ -0,0 +1,104 @@ +"""B6 — a real bet on the local regtest: deposit, reservation, signature by the worker, broadcast to the node, +the change coming back, and the verdict of a refused transaction (BET-30..BET-33, DEP-22, TX-04).""" + +import asyncio +import time +from datetime import timedelta +from uuid import uuid4 + +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone +from embit.transaction import Transaction +from plm_engine.electrum.client import ElectrumEndpoint +from plm_engine.network import REGTEST + +from apps.lottery import bets, chain, deposits, keys +from apps.lottery.balances import balance_of +from apps.lottery.electrum_chain import ElectrumChain, Outcome +from apps.lottery.ledger import ChainLedger +from apps.lottery.models import ChainTransaction, Deposit, Movement, Participation, Round, Utxo + +pytestmark = pytest.mark.django_db +PLM = 100_000_000 +BET = 10 * PLM + + +@pytest.fixture +def network(electrum_endpoint): + host, port = electrum_endpoint + return ElectrumChain([ElectrumEndpoint(host, port, False)], REGTEST, allow_no_quorum=True) + + +@pytest.fixture +def funded_player(settings, tmp_path, regtest_node, electrum_endpoint, network): + """A player whose 30 PLM deposit has been through 6 confirmations and the scan.""" + settings.LEDGER_BACKEND = "chain" + settings.PLM_ACCOUNT_XPUB = keys.generate_keys(tmp_path / "keys", settings.PLM_PROFILE) + settings.PLM_MASTER_KEY_FILE = str(tmp_path / "keys" / "master.key") + settings.PLM_KEY_ENCRYPTION_KEY_FILE = str(tmp_path / "keys" / "master.encryption.key") + settings.PROCESS_ROLE = "worker" + user = get_user_model().objects.create_user("giocatore", password="x" * 12) + address = chain.assign_deposit_address(user).address + regtest_node.fund(address, 30) + regtest_node.mine(5) + for _ in range(80): + deposits.scan_deposits(network) + if balance_of(user).spendable_sats: + break + time.sleep(0.25) + assert balance_of(user).spendable_sats == 30 * PLM + return user, address + + +def open_round(): + round_ = Round.objects.create(opened_at=timezone.now() - timedelta(seconds=30), bet_amount_sats=BET) + round_.pool_address = chain.derive_pool_address(round_.pk) + round_.save(update_fields=["pool_address"]) + return round_ + + +def test_a_real_bet_goes_from_the_web_to_the_node(regtest_node, network, funded_player): + user, address = funded_player + round_ = open_round() + ChainLedger().place_bet(user, round_.pk, request_id=uuid4()) # the web: admits and reserves, signs nothing + tx = ChainTransaction.objects.get() + assert tx.status == "building" and tx.current_txid is None + + done = bets.process_bets(network) # the worker: signs, saves, sends + assert (done["signed"], done["accepted"]) == (1, 1) + tx.refresh_from_db() + assert tx.status == "pending" + + entry = regtest_node.rpc("getmempoolentry", tx.current_txid) # the node really has it + assert entry["vsize"] > 0 + raw = regtest_node.rpc("getrawtransaction", tx.current_txid) + pool_out, change_out = Transaction.parse(bytes.fromhex(raw)).vout + participation = Participation.objects.get() + assert pool_out.value == participation.bet_amount_sats and pool_out.value < BET + script_pubkey = regtest_node.rpc("getrawtransaction", tx.current_txid, True)["vout"][0]["scriptPubKey"] + assert (script_pubkey.get("address") or script_pubkey["addresses"][0]) == round_.pool_address # the node's own reading + assert change_out.value == 20 * PLM + + # sending the same transaction again is "already known": still accepted, never a refusal + verdict = asyncio.run(network.broadcast(tx.raw_hex, tx.current_txid)) + assert verdict.outcome is Outcome.ACCEPTED + + regtest_node.mine(1) + for _ in range(80): # the change comes back to the player's address: spendable money, not a deposit + deposits.scan_deposits(network) + if Utxo.objects.filter(txid=tx.current_txid).exists(): + break + time.sleep(0.25) + change = Utxo.objects.get(txid=tx.current_txid) + assert change.amount_sats == 20 * PLM and not Deposit.objects.filter(utxo=change).exists() + assert Movement.objects.filter(kind="deposit").count() == 1 # still the one real deposit + assert balance_of(user).spendable_sats == 20 * PLM + + +def test_a_transaction_the_chain_cannot_accept_is_refused_with_proof(network): + # one input that does not exist, one output: the node refuses and no server knows the txid + raw = "01000000" + "01" + "11" * 32 + "00000000" + "00" + "ffffffff" + "01" + "00ca9a3b00000000" + "00" + "00000000" + txid = Transaction.parse(bytes.fromhex(raw)).txid().hex() + verdict = asyncio.run(network.broadcast(raw, txid)) + assert verdict.outcome is Outcome.REFUSED and verdict.reason