B5: real deposits on the ORM, Electrum scan with quorum and merkle proof, configurable threshold (DEP-20..23, ELC-31)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
davideandClaude Sonnet 5.5 committed 2026-11-16 16:09:36 +01:00
1 parent f1d09075a1
commit 2acd4ccd1c
21 files changed
+1223 -49

No files matched your search

+24 -1
View File
@@ -5,7 +5,7 @@ from django.contrib import admin
from django.db import transaction
from . import audit
from .models import AuditEntry, ChainTransaction, Movement, Participation, Report, Round
from .models import AuditEntry, ChainTransaction, LotteryConfig, Movement, Participation, Report, Round
class ReadOnlyAdmin(admin.ModelAdmin):
@@ -75,3 +75,26 @@ class ReportAdmin(admin.ModelAdmin):
obj.save(update_fields=["status", "status_changed_at"])
audit.record("report_status_changed", actor=request.user, subject_type="report", subject_id=obj.pk,
payload={"from": before, "to": obj.status})
@admin.register(LotteryConfig)
class LotteryConfigAdmin(admin.ModelAdmin):
"""The operational parameters (DAT-77). The three pause switches are not here: they go through the console's
commands, which ask for a reason (WEB-43). A change is audited with what changed."""
fields = ("fee_address", "bet_amount_sats", "round_duration_seconds", "round_cooldown_seconds",
"draw_confirmations", "deposit_confirmations", "max_round_pool_sats")
def has_add_permission(self, request):
return False
def has_delete_permission(self, request, obj=None):
return False
def save_model(self, request, obj, form, change):
changes = {name: [form.initial.get(name), getattr(obj, name)] for name in form.changed_data}
with transaction.atomic():
super().save_model(request, obj, form, change)
if changes:
audit.record("config_changed", actor=request.user, subject_type="config", subject_id=obj.pk,
payload={"changed": changes})
+185
View File
@@ -0,0 +1,185 @@
"""Real deposits on the Django site (roadmap B5, DEP-20..DEP-23).
A scan is three phases (DEP-20): `plan` reads the database, `ElectrumChain.observe` asks the servers with no
database open, `apply_observations` writes only what was verified, re-checking each coin as it saves. What
happens after a credit — coins spent elsewhere, reorgs deeper than the threshold — is B7.
"""
import logging
from dataclasses import dataclass, field
from datetime import datetime
from django.db import transaction
from django.utils import timezone
from plm_engine.psbt_builder import DUST_LIMIT_SATS
from . import audit
from .models import Deposit, DepositAddress, LotteryConfig, Movement, TransactionVersion, Utxo
logger = logging.getLogger(__name__)
COINBASE_MATURITY = 120 # DEP-21: PLM coinbase maturity (palladiumcore consensus.h)
@dataclass(frozen=True)
class Policy:
"""What phase 1 hands to the network phase."""
addresses: tuple[str, ...]
own_txids: frozenset[str]
deposit_confirmations: int
credited: frozenset[tuple[str, int]] # outpoints already credited: never looked at again
waiting: dict[str, frozenset[tuple[str, int]]] # per address, the deposits seen but not yet credited
@dataclass(frozen=True)
class Seen:
"""An unspent output as the servers list it."""
txid: str
vout: int
value: int
height: int # 0 = mempool
@dataclass
class Observation:
"""What phase 2 found at one address. `entries` is None when no server gave a usable answer: nothing is
concluded from silence. `verified` holds the outpoints whose credit the other servers corroborated and whose
merkle proof held; `absent_confirmed` those the other servers also do not list any more."""
entries: list[Seen] | None
verified: set[tuple[str, int]] = field(default_factory=set)
absent_confirmed: set[tuple[str, int]] = field(default_factory=set)
coinbase: set[str] = field(default_factory=set)
@dataclass
class ScanResult:
tip: int | None
addresses: dict[str, Observation]
def threshold_for(txid: str, policy: Policy, coinbase: set[str]) -> int:
"""DEP-21: 1 for the platform's own transactions, 120 for a coinbase, else the configured threshold."""
if txid in policy.own_txids:
return 1
if txid in coinbase:
return COINBASE_MATURITY
return policy.deposit_confirmations
def plan() -> Policy:
"""Phase 1: everything the network phase needs, read in one go."""
return Policy(
addresses=tuple(DepositAddress.objects.order_by("derivation_index").values_list("address", flat=True)),
own_txids=frozenset(TransactionVersion.objects.values_list("txid", flat=True)),
deposit_confirmations=LotteryConfig.get().deposit_confirmations,
credited=frozenset(Utxo.objects.filter(deposit__status=Deposit.Status.CREDITED).values_list("txid", "vout")),
waiting=_waiting_by_address(),
)
def _waiting_by_address() -> dict[str, frozenset[tuple[str, int]]]:
grouped: dict[str, set[tuple[str, int]]] = {}
rows = Utxo.objects.filter(deposit__status__in=[Deposit.Status.DETECTED, Deposit.Status.ORPHANED])
for address, txid, vout in rows.values_list("address", "txid", "vout"):
grouped.setdefault(address, set()).add((txid, vout))
return {address: frozenset(outpoints) for address, outpoints in grouped.items()}
def confirmations(tip: int, height: int) -> int:
return tip - height + 1
def apply_observations(result: ScanResult, policy: Policy, now: datetime | None = None) -> dict[str, int]:
"""Phase 3: write what was verified. Returns counters (`detected`, `credited`, `orphaned`, `reinstated`)."""
now = now or timezone.now()
done = {"detected": 0, "credited": 0, "orphaned": 0, "reinstated": 0}
if result.tip is None:
return done
for address, observation in result.addresses.items():
if observation.entries is None:
continue
owner = DepositAddress.objects.filter(address=address).select_related("user").first()
if owner is None:
continue
listed = {(e.txid, e.vout): e for e in observation.entries}
for entry in observation.entries:
if entry.value < DUST_LIMIT_SATS or entry.height <= 0:
continue # DEP-21
_apply_entry(owner, entry, result.tip, policy, observation, now, done)
# DEP-23: a waiting deposit that the servers agree is gone
for deposit in Deposit.objects.filter(utxo__address=address, status=Deposit.Status.DETECTED).select_related("utxo"):
outpoint = (deposit.utxo.txid, deposit.utxo.vout)
if outpoint not in listed and outpoint in observation.absent_confirmed:
_orphan(deposit, done)
return done
def _apply_entry(owner, entry: Seen, tip: int, policy: Policy, observation: Observation, now, done) -> None:
outpoint = (entry.txid, entry.vout)
have = confirmations(tip, entry.height)
need = threshold_for(entry.txid, policy, observation.coinbase)
ready = have >= need and outpoint in observation.verified
with transaction.atomic():
utxo = Utxo.objects.filter(txid=entry.txid, vout=entry.vout).first()
deposit = Deposit.objects.select_for_update().filter(utxo=utxo).first() if utxo is not None else None
if deposit is None:
if utxo is not None:
return # an output that is not a deposit (e.g. one of our own): not this module's business
utxo = Utxo.objects.create(txid=entry.txid, vout=entry.vout, amount_sats=entry.value, address=owner.address,
user=owner.user, confirmed_height=entry.height, created_at=now)
movement = Movement.objects.create(user=owner.user, kind=Movement.Kind.DEPOSIT, status=Movement.Status.PENDING,
amount_sats=entry.value, created_at=now)
deposit = Deposit.objects.create(user=owner.user, utxo=utxo, movement=movement, detected_at=now)
done["detected"] += 1
elif deposit.status == Deposit.Status.ORPHANED:
deposit.status = Deposit.Status.DETECTED
deposit.save(update_fields=["status"])
Movement.objects.filter(pk=deposit.movement_id).update(status=Movement.Status.PENDING)
done["reinstated"] += 1
if deposit.status == Deposit.Status.CREDITED:
return
if utxo.confirmed_height != entry.height: # the coin moved to another block: it counts from there
utxo.confirmed_height = entry.height
utxo.save(update_fields=["confirmed_height"])
if not ready:
return
movement = deposit.movement
movement.status, movement.confirmed_at = Movement.Status.CONFIRMED, now
movement.save(update_fields=["status", "confirmed_at"])
deposit.status, deposit.credited_at = Deposit.Status.CREDITED, now
deposit.save(update_fields=["status", "credited_at"])
audit.record("deposit_credited", actor_label="worker", subject_type="deposit", subject_id=deposit.pk,
payload={"txid": entry.txid, "vout": entry.vout, "amount_sats": entry.value,
"height": entry.height, "confirmations": have, "threshold": need})
done["credited"] += 1
def _orphan(deposit: Deposit, done) -> None:
with transaction.atomic():
fresh = Deposit.objects.select_for_update().get(pk=deposit.pk)
if fresh.status != Deposit.Status.DETECTED:
return # credited or orphaned meanwhile
fresh.status = Deposit.Status.ORPHANED
fresh.save(update_fields=["status"])
Movement.objects.filter(pk=fresh.movement_id).update(status=Movement.Status.FAILED)
audit.record("deposit_orphaned", actor_label="worker", subject_type="deposit", subject_id=fresh.pk,
payload={"txid": fresh.utxo.txid, "vout": fresh.utxo.vout})
done["orphaned"] += 1
def scan_deposits(chain, now: datetime | None = None) -> dict[str, int]:
"""One scan. `chain.observe(policy)` is a coroutine function (`ElectrumChain`) or any object whose `observe`
returns a `ScanResult` — the tests use the latter."""
import asyncio
import inspect
policy = plan()
if not policy.addresses:
return {"detected": 0, "credited": 0, "orphaned": 0, "reinstated": 0}
result = chain.observe(policy)
if inspect.isawaitable(result):
result = asyncio.run(result)
return apply_observations(result, policy, now)
+224
View File
@@ -0,0 +1,224 @@
"""The network phase of a deposit scan (roadmap B5, DEP-20, ELC-14, ELC-17, ELC-26, ELC-31).
Nothing here touches the database. `observe` takes the `Policy` read in phase 1 and returns what the servers
say, already corroborated: the active server is the first one that answers *and* has the genesis of the network
profile; every other configured server is asked, over a fresh connection with its own time limit, whether it
agrees. Silence is never an answer (ELC-14): a server that does not respond does not count.
"""
import asyncio
import hashlib
import logging
from collections.abc import Awaitable, Callable
from embit.transaction import Transaction
from plm_engine.electrum.client import ElectrumClient, ElectrumEndpoint
from plm_engine.electrum.merkle import proof_matches_header
from plm_engine.electrum.scripthash import address_to_scripthash
from plm_engine.headers import header_meets_its_own_target
from plm_engine.network import Profile
from plm_engine.psbt_builder import DUST_LIMIT_SATS
from .deposits import Observation, Policy, ScanResult, Seen, confirmations, threshold_for
logger = logging.getLogger(__name__)
CONNECT_TIMEOUT = 10.0
ANSWER_TIMEOUT = 10.0
def _seen(raw: dict) -> Seen | None:
try:
return Seen(txid=str(raw["tx_hash"]), vout=int(raw["tx_pos"]), value=int(raw["value"]), height=int(raw["height"]))
except (KeyError, TypeError, ValueError):
return None
class ElectrumChain:
def __init__(
self,
endpoints: list[ElectrumEndpoint],
profile: Profile,
allow_no_quorum: bool = False,
client_factory: Callable[[ElectrumEndpoint], ElectrumClient] | None = None,
):
self.endpoints = endpoints
self.profile = profile
self.allow_no_quorum = allow_no_quorum
self._factory = client_factory or (
lambda e: ElectrumClient(e.host, e.port, e.use_ssl, e.pin, genesis_hash=profile.genesis_hash)
)
self.active_endpoint: ElectrumEndpoint | None = None
# -- connections ------------------------------------------------------------------------------
async def _open(self, endpoint: ElectrumEndpoint) -> ElectrumClient | None:
client = self._factory(endpoint)
try:
await asyncio.wait_for(client.connect(), CONNECT_TIMEOUT)
except Exception as error: # unreachable, wrong chain, bad pin: all are "this server is not usable now"
logger.warning("electrum %s unusable: %s", endpoint, error)
return None
return client
async def _connect_active(self) -> ElectrumClient | None:
for endpoint in self.endpoints: # rotation: the first usable one in configuration order
client = await self._open(endpoint)
if client is not None:
self.active_endpoint = endpoint
return client
return None
async def _ask_others(self, ask: Callable[[ElectrumClient], Awaitable]) -> list:
"""`ask` on every server but the active one, concurrently, each over a fresh connection. `None` = no answer."""
async def one(endpoint: ElectrumEndpoint):
client = await self._open(endpoint)
if client is None:
return None
try:
return await asyncio.wait_for(ask(client), ANSWER_TIMEOUT)
except Exception as error:
logger.warning("electrum %s did not answer: %s", endpoint, error)
return None
finally:
await client.close()
others = [e for e in self.endpoints if e != self.active_endpoint]
return list(await asyncio.gather(*(one(e) for e in others)))
def majority(self, answers: list) -> bool:
"""ELC-14. Answers are True (agrees), False (disagrees) or None (silent). At least two answered and a strict
majority of those agrees; with the development exception, no other server at all passes and any strict majority
of even one answer does."""
answered = [a for a in answers if a is not None]
if self.allow_no_quorum:
if not self.endpoints or len(self.endpoints) == 1:
return True
return bool(answered) and sum(bool(a) for a in answered) * 2 > len(answered)
return len(answered) >= 2 and sum(bool(a) for a in answered) * 2 > len(answered)
# -- what is asked ------------------------------------------------------------------------------
async def _tip(self, active: ElectrumClient) -> tuple[int, str] | None:
"""The active server's tip, accepted only with work valid for the profile and agreed by the others."""
reply = await active.subscribe_headers()
try:
height, header = int(reply["height"]), str(reply["hex"])
except (KeyError, TypeError, ValueError):
return None
if not header_meets_its_own_target(header, self.profile.pow_limit):
logger.warning("the tip header of %s has no valid work", self.active_endpoint)
return None
async def same_header(client: ElectrumClient) -> bool:
return await client.get_block_header(height) == header
if not self.majority(await self._ask_others(same_header)):
return None
return height, header
async def _unspent(self, client: ElectrumClient, address: str) -> list[Seen] | None:
try:
raw = await asyncio.wait_for(client.listunspent(address_to_scripthash(address)), ANSWER_TIMEOUT)
except Exception as error:
logger.warning("listunspent for %s failed: %s", address, error)
return None
if not isinstance(raw, list):
return None
seen = [_seen(item) for item in raw if isinstance(item, dict)]
return None if any(item is None for item in seen) else seen
async def is_coinbase(self, client: ElectrumClient, txid: str) -> bool | None:
"""From the raw transaction, accepted only if its hash is the txid (DEP-21); None = could not tell."""
try:
raw = await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT)
tx = Transaction.parse(bytes.fromhex(str(raw)))
if tx.txid().hex() != txid:
return None
except Exception as error:
logger.warning("cannot read transaction %s: %s", txid, error)
return None
return len(tx.vin) == 1 and tx.vin[0].txid == bytes(32)
async def _credit_verified(self, active: ElectrumClient, address: str, entry: Seen, tip: int, need: int) -> bool:
"""ELC-17 (the others list the coin, same value, deep enough) and ELC-26 (a merkle proof from the others
matches a header of the active server that has valid work)."""
scripthash = address_to_scripthash(address)
deepest = tip - need + 1
async def lists_it(client: ElectrumClient) -> bool:
for raw in await client.listunspent(scripthash):
other = _seen(raw) if isinstance(raw, dict) else None
if other and (other.txid, other.vout) == (entry.txid, entry.vout):
return other.value == entry.value and 0 < other.height <= deepest
return False
if not self.majority(await self._ask_others(lists_it)):
return False
try:
header = await asyncio.wait_for(active.get_block_header(entry.height), ANSWER_TIMEOUT)
except Exception:
return False
if not header_meets_its_own_target(str(header), self.profile.pow_limit):
return False
async def proves_it(client: ElectrumClient) -> bool:
return proof_matches_header(entry.txid, entry.height, await client.get_merkle(entry.txid, entry.height), str(header))
return self.majority(await self._ask_others(proves_it))
async def _absent_confirmed(self, address: str, outpoint: tuple[str, int]) -> bool:
scripthash = address_to_scripthash(address)
async def lacks_it(client: ElectrumClient) -> bool:
listed = {(r.get("tx_hash"), r.get("tx_pos")) for r in await client.listunspent(scripthash) if isinstance(r, dict)}
return outpoint not in listed
return self.majority(await self._ask_others(lacks_it))
# -- the scan ----------------------------------------------------------------------------------------
async def observe(self, policy: Policy) -> ScanResult:
active = await self._connect_active()
if active is None:
return ScanResult(None, {})
try:
tip = await self._tip(active)
if tip is None:
return ScanResult(None, {})
result = ScanResult(tip[0], {})
for address in policy.addresses:
result.addresses[address] = await self._observe_address(active, address, tip[0], policy)
return result
except Exception: # a scan that fails writes nothing; the next one starts again
logger.exception("deposit scan failed")
return ScanResult(None, {})
finally:
await active.close()
async def _observe_address(self, active: ElectrumClient, address: str, tip: int, policy: Policy) -> Observation:
entries = await self._unspent(active, address)
observation = Observation(entries)
if entries is None:
return observation
listed = {(e.txid, e.vout) for e in entries}
for entry in entries:
outpoint = (entry.txid, entry.vout)
if entry.value < DUST_LIMIT_SATS or entry.height <= 0 or outpoint in policy.credited:
continue
have = confirmations(tip, entry.height)
if have >= policy.deposit_confirmations and entry.txid not in policy.own_txids:
coinbase = await self.is_coinbase(active, entry.txid)
if coinbase is None:
continue # unreadable: it cannot be credited as an ordinary deposit by mistake
if coinbase:
observation.coinbase.add(entry.txid)
need = threshold_for(entry.txid, policy, observation.coinbase)
if have >= need and await self._credit_verified(active, address, entry, tip, need):
observation.verified.add(outpoint)
for outpoint in policy.waiting.get(address, frozenset()) - listed:
if await self._absent_confirmed(address, outpoint):
observation.absent_confirmed.add(outpoint)
return observation
@@ -15,6 +15,29 @@ logger = logging.getLogger(__name__)
TICK_SECONDS = 5
_last_scan = 0.0
SCAN_SECONDS = 10
def scan_chain_deposits() -> None:
"""DEP-20: every few seconds, on a real ledger only."""
import time
from django.conf import settings
from apps.lottery import deposits
from apps.lottery.electrum_chain import ElectrumChain
global _last_scan
if time.monotonic() - _last_scan < SCAN_SECONDS:
return
_last_scan = time.monotonic()
chain = ElectrumChain(settings.ELECTRUM_ENDPOINTS, settings.PLM_PROFILE, settings.ELECTRUM_ALLOW_NO_QUORUM)
done = deposits.scan_deposits(chain)
if any(done.values()):
logger.info("deposit scan: %s", done)
def tick() -> None:
"""One pass over everything the worker is responsible for."""
from django.conf import settings
@@ -24,6 +47,8 @@ def tick() -> None:
console.beat() # WEB-31: proof of life, before the work that might fail
if settings.LEDGER_BACKEND == "simulated":
simulation.settle()
else:
scan_chain_deposits() # B5
lifecycle.tick()
@@ -0,0 +1,22 @@
# Generated by Django 5.2.19 on 2026-11-16 14:58
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('lottery', '0005_transaction_versions_append_only'),
]
operations = [
migrations.AddField(
model_name='lotteryconfig',
name='deposit_confirmations',
field=models.PositiveSmallIntegerField(default=6),
),
migrations.AddConstraint(
model_name='lotteryconfig',
constraint=models.CheckConstraint(condition=models.Q(('deposit_confirmations__gte', 1), ('deposit_confirmations__lte', 100)), name='lottery_config_deposit_confirmations_range'),
),
]
+7
View File
@@ -31,6 +31,9 @@ class LotteryConfig(models.Model):
round_duration_seconds = models.PositiveIntegerField(default=600)
round_cooldown_seconds = models.PositiveIntegerField(default=30)
draw_confirmations = models.PositiveSmallIntegerField(default=3)
# DAT-62, DEP-21: confirmations a deposit needs before it is credited; read at every scan, so a change
# applies at once to the credits that follow.
deposit_confirmations = models.PositiveSmallIntegerField(default=6)
max_round_pool_sats = models.BigIntegerField(default=0) # 0 = no cap
# ADR-57: three distinct commands.
@@ -60,6 +63,10 @@ class LotteryConfig(models.Model):
condition=Q(draw_confirmations__gte=1, draw_confirmations__lte=20),
name="lottery_config_draw_confirmations_range",
),
models.CheckConstraint(
condition=Q(deposit_confirmations__gte=1, deposit_confirmations__lte=100),
name="lottery_config_deposit_confirmations_range",
),
]
@classmethod