Files
plm-lottery/apps/lottery/electrum_chain.py
T

379 lines
18 KiB
Python

"""The network phase of a deposit scan (roadmap B5, DEP-20, ELC-14, ELC-17, ELC-26, ELC-31).
Nothing here touches the database. `observe` takes the `Policy` read in phase 1 and returns what the servers
say, already corroborated: the active server is the first one that answers *and* has the genesis of the network
profile; every other configured server is asked, over a fresh connection with its own time limit, whether it
agrees. Silence is never an answer (ELC-14): a server that does not respond does not count.
"""
import asyncio
import hashlib
import logging
from collections.abc import Awaitable, Callable
from embit.transaction import Transaction
from enum import Enum
from typing import NamedTuple
from plm_engine.electrum.client import ElectrumClient, ElectrumEndpoint, ElectrumRpcError
from plm_engine.electrum.merkle import proof_matches_header
from plm_engine.electrum.scripthash import address_to_scripthash
from plm_engine.headers import header_meets_its_own_target
from plm_engine.network import Profile
from plm_engine.psbt_builder import DUST_LIMIT_SATS
from .deposits import Observation, Policy, ScanResult, Seen, confirmations, threshold_for
logger = logging.getLogger(__name__)
CONNECT_TIMEOUT = 10.0
ANSWER_TIMEOUT = 10.0
class Outcome(Enum):
ACCEPTED = "accepted"
REFUSED = "refused"
UNCERTAIN = "uncertain"
class Verdict(NamedTuple):
outcome: Outcome
reason: str
# Wordings by which a node says it already has the transaction. Recognising one skips the verification; not
# recognising one never releases anything, it only costs one more round of questions (BET-32, TX-04).
_ALREADY_KNOWN = ("already known", "already in mempool", "already in the mempool", "txn-already-known",
"txn-already-in-mempool", "already in block chain", "already in blockchain")
def _seen(raw: dict) -> Seen | None:
try:
return Seen(txid=str(raw["tx_hash"]), vout=int(raw["tx_pos"]), value=int(raw["value"]), height=int(raw["height"]))
except (KeyError, TypeError, ValueError):
return None
class ElectrumChain:
def __init__(
self,
endpoints: list[ElectrumEndpoint],
profile: Profile,
allow_no_quorum: bool = False,
client_factory: Callable[[ElectrumEndpoint], ElectrumClient] | None = None,
):
self.endpoints = endpoints
self.profile = profile
self.allow_no_quorum = allow_no_quorum
self._factory = client_factory or (
lambda e: ElectrumClient(e.host, e.port, e.use_ssl, e.pin, genesis_hash=profile.genesis_hash)
)
self.active_endpoint: ElectrumEndpoint | None = None
# -- connections ------------------------------------------------------------------------------
async def _open(self, endpoint: ElectrumEndpoint) -> ElectrumClient | None:
client = self._factory(endpoint)
try:
await asyncio.wait_for(client.connect(), CONNECT_TIMEOUT)
except Exception as error: # unreachable, wrong chain, bad pin: all are "this server is not usable now"
logger.warning("electrum %s unusable: %s", endpoint, error)
return None
return client
async def _connect_active(self) -> ElectrumClient | None:
for endpoint in self.endpoints: # rotation: the first usable one in configuration order
client = await self._open(endpoint)
if client is not None:
self.active_endpoint = endpoint
return client
return None
async def _ask_others(self, ask: Callable[[ElectrumClient], Awaitable]) -> list:
"""`ask` on every server but the active one, concurrently, each over a fresh connection. `None` = no answer."""
async def one(endpoint: ElectrumEndpoint):
client = await self._open(endpoint)
if client is None:
return None
try:
return await asyncio.wait_for(ask(client), ANSWER_TIMEOUT)
except Exception as error:
logger.warning("electrum %s did not answer: %s", endpoint, error)
return None
finally:
await client.close()
others = [e for e in self.endpoints if e != self.active_endpoint]
return list(await asyncio.gather(*(one(e) for e in others)))
def majority(self, answers: list) -> bool:
"""ELC-14. Answers are True (agrees), False (disagrees) or None (silent). At least two answered and a strict
majority of those agrees; with the development exception, no other server at all passes and any strict majority
of even one answer does."""
answered = [a for a in answers if a is not None]
if self.allow_no_quorum:
if not self.endpoints or len(self.endpoints) == 1:
return True
return bool(answered) and sum(bool(a) for a in answered) * 2 > len(answered)
return len(answered) >= 2 and sum(bool(a) for a in answered) * 2 > len(answered)
# -- what is asked ------------------------------------------------------------------------------
async def _tip(self, active: ElectrumClient) -> tuple[int, str] | None:
"""The active server's tip, accepted only with work valid for the profile and agreed by the others."""
reply = await active.subscribe_headers()
try:
height, header = int(reply["height"]), str(reply["hex"])
except (KeyError, TypeError, ValueError):
return None
if not header_meets_its_own_target(header, self.profile.pow_limit):
logger.warning("the tip header of %s has no valid work", self.active_endpoint)
return None
async def same_header(client: ElectrumClient) -> bool:
return await client.get_block_header(height) == header
if not self.majority(await self._ask_others(same_header)):
return None
return height, header
async def _unspent(self, client: ElectrumClient, address: str) -> list[Seen] | None:
try:
raw = await asyncio.wait_for(client.listunspent(address_to_scripthash(address)), ANSWER_TIMEOUT)
except Exception as error:
logger.warning("listunspent for %s failed: %s", address, error)
return None
if not isinstance(raw, list):
return None
seen = [_seen(item) for item in raw if isinstance(item, dict)]
return None if any(item is None for item in seen) else seen
async def is_coinbase(self, client: ElectrumClient, txid: str) -> bool | None:
"""From the raw transaction, accepted only if its hash is the txid (DEP-21); None = could not tell."""
try:
raw = await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT)
tx = Transaction.parse(bytes.fromhex(str(raw)))
if tx.txid().hex() != txid:
return None
except Exception as error:
logger.warning("cannot read transaction %s: %s", txid, error)
return None
return len(tx.vin) == 1 and tx.vin[0].txid == bytes(32)
async def _credit_verified(self, active: ElectrumClient, address: str, entry: Seen, tip: int, need: int) -> bool:
"""ELC-17 (the others list the coin, same value, deep enough) and ELC-26 (a merkle proof from the others
matches a header of the active server that has valid work)."""
scripthash = address_to_scripthash(address)
deepest = tip - need + 1
async def lists_it(client: ElectrumClient) -> bool:
for raw in await client.listunspent(scripthash):
other = _seen(raw) if isinstance(raw, dict) else None
if other and (other.txid, other.vout) == (entry.txid, entry.vout):
return other.value == entry.value and 0 < other.height <= deepest
return False
if not self.majority(await self._ask_others(lists_it)):
return False
return await self._proof_holds(active, entry.txid, entry.height)
async def _proof_holds(self, active: ElectrumClient, txid: str, height: int) -> bool:
"""ELC-26, TX-22: a header of the active server at `height` with valid work for the profile, and a merkle
proof from the others (by majority) that puts `txid` in that block."""
try:
header = await asyncio.wait_for(active.get_block_header(height), ANSWER_TIMEOUT)
except Exception:
return False
if not header_meets_its_own_target(str(header), self.profile.pow_limit):
return False
async def proves_it(client: ElectrumClient) -> bool:
return proof_matches_header(txid, height, await client.get_merkle(txid, height), str(header))
return self.majority(await self._ask_others(proves_it))
async def _absent_confirmed(self, address: str, outpoint: tuple[str, int]) -> bool:
scripthash = address_to_scripthash(address)
async def lacks_it(client: ElectrumClient) -> bool:
listed = {(r.get("tx_hash"), r.get("tx_pos")) for r in await client.listunspent(scripthash) if isinstance(r, dict)}
return outpoint not in listed
return self.majority(await self._ask_others(lacks_it))
# -- confirmations and presence (TX-30, TX-33) ----------------------------------------------------------------
@staticmethod
async def _history(client: ElectrumClient, address: str) -> list[tuple[str, int]] | None:
try:
raw = await asyncio.wait_for(client.get_history(address_to_scripthash(address)), ANSWER_TIMEOUT)
except Exception as error:
logger.warning("history of %s failed: %s", address, error)
return None
try:
return [(str(item["tx_hash"]), int(item["height"])) for item in raw]
except (KeyError, TypeError, ValueError):
return None
async def observe_confirmations(self, wanted: list[tuple[int, str, frozenset[str]]]) -> dict[int, tuple[str, int]]:
"""`wanted` is `(row id, spending address, every txid of the row)`. Returns, per row, the version that is in a
block *and* proven (TX-30): `{row id: (txid, height)}`. An address that cannot be read blocks nobody else."""
active = await self._connect_active()
if active is None:
return {}
try:
histories: dict[str, list[tuple[str, int]] | None] = {}
confirmed: dict[int, tuple[str, int]] = {}
for row_id, address, txids in wanted:
if address not in histories:
histories[address] = await self._history(active, address)
history = histories[address]
if history is None:
continue
for txid, height in history:
if txid in txids and height > 0 and await self._proof_holds(active, txid, height):
confirmed[row_id] = (txid, height)
break
return confirmed
except Exception:
logger.exception("confirmation poll failed")
return {}
finally:
await active.close()
async def versions_known(self, address: str, txids: frozenset[str]) -> tuple[set[str], bool]:
"""TX-33: which of `txids` any server of the network has seen (mempool or block), and whether the network
answered enough for the *absence* of the rest to mean something (the active server and the quorum)."""
active = await self._connect_active()
if active is None:
return set(), False
try:
mine = await self._history(active, address)
async def theirs(client: ElectrumClient):
history = await self._history(client, address)
return None if history is None else {txid for txid, _ in history if txid in txids}
others = await self._ask_others(theirs)
known = ({txid for txid, _ in mine if txid in txids} if mine is not None else set())
for answer in others:
if answer:
known |= answer
# an absence is a statement only if the active server answered and the others agree among themselves
complete = mine is not None and self.majority([a is not None for a in others])
return known, complete
finally:
await active.close()
# -- broadcast (BET-32) -----------------------------------------------------------------------------------
async def transaction_known(self, active: ElectrumClient, txid: str) -> bool | None:
"""True if any server returns the transaction; False only if the active server and the quorum of the
others all say they do not know it; None if there were not enough answers to say."""
not_found = ("no such mempool", "no such blockchain", "not found", "unknown transaction")
async def ask(client: ElectrumClient):
try:
await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT)
return True
except ElectrumRpcError as error: # the server answered: it does not know it
return False if any(m in error.message.lower() for m in not_found) else None
except Exception:
return None
mine = await ask(active)
others = await self._ask_others(ask)
if mine is True or any(a is True for a in others):
return True
if mine is False and self.majority([a is False if a is not None else None for a in others]):
return False
return None
async def broadcast(self, raw_tx_hex: str, txid: str) -> Verdict:
"""Send `raw_tx_hex` through the active server and say what happened (BET-32). Only a proven refusal frees coins."""
active = await self._connect_active()
if active is None:
return Verdict(Outcome.UNCERTAIN, "no Electrum server is reachable")
try:
try:
await asyncio.wait_for(active.broadcast(raw_tx_hex), ANSWER_TIMEOUT)
except ElectrumRpcError as error:
reason = error.message
known = True if any(m in reason.lower() for m in _ALREADY_KNOWN) else await self.transaction_known(active, txid)
if known is True:
await self._fan_out(raw_tx_hex)
return Verdict(Outcome.ACCEPTED, reason)
if known is False:
return Verdict(Outcome.REFUSED, reason)
logger.warning("broadcast of %s was refused (%s) but the refusal could not be verified", txid, reason)
return Verdict(Outcome.UNCERTAIN, reason)
except Exception as error: # timeout, dropped connection: the server may have relayed it
logger.warning("broadcast of %s ended without an answer (%r): uncertain", txid, error)
return Verdict(Outcome.UNCERTAIN, str(error) or type(error).__name__)
await self._fan_out(raw_tx_hex)
return Verdict(Outcome.ACCEPTED, "")
finally:
await active.close()
async def _fan_out(self, raw_tx_hex: str) -> None:
"""TX-21: after the active server accepted, tell the others too (best effort)."""
async def send(client: ElectrumClient):
try:
await client.broadcast(raw_tx_hex)
except ElectrumRpcError:
pass # "already known" and the like: that is what we wanted
return True
try:
await self._ask_others(send)
except Exception:
logger.exception("fan-out of an accepted transaction failed")
# -- the scan ----------------------------------------------------------------------------------------
async def observe(self, policy: Policy) -> ScanResult:
active = await self._connect_active()
if active is None:
return ScanResult(None, {})
try:
tip = await self._tip(active)
if tip is None:
return ScanResult(None, {})
result = ScanResult(tip[0], {})
for address in policy.addresses:
result.addresses[address] = await self._observe_address(active, address, tip[0], policy)
return result
except Exception: # a scan that fails writes nothing; the next one starts again
logger.exception("deposit scan failed")
return ScanResult(None, {})
finally:
await active.close()
async def _observe_address(self, active: ElectrumClient, address: str, tip: int, policy: Policy) -> Observation:
entries = await self._unspent(active, address)
observation = Observation(entries)
if entries is None:
return observation
listed = {(e.txid, e.vout) for e in entries}
for entry in entries:
outpoint = (entry.txid, entry.vout)
if entry.value < DUST_LIMIT_SATS or entry.height <= 0 or outpoint in policy.credited:
continue
have = confirmations(tip, entry.height)
if have >= policy.deposit_confirmations and entry.txid not in policy.own_txids:
coinbase = await self.is_coinbase(active, entry.txid)
if coinbase is None:
continue # unreadable: it cannot be credited as an ordinary deposit by mistake
if coinbase:
observation.coinbase.add(entry.txid)
need = threshold_for(entry.txid, policy, observation.coinbase)
if have >= need and await self._credit_verified(active, address, entry, tip, need):
observation.verified.add(outpoint)
for outpoint in policy.waiting.get(address, frozenset()) - listed:
if await self._absent_confirmed(address, outpoint):
observation.absent_confirmed.add(outpoint)
return observation