"""The network phase of a deposit scan (roadmap B5, DEP-20, ELC-14, ELC-17, ELC-26, ELC-31). Nothing here touches the database. `observe` takes the `Policy` read in phase 1 and returns what the servers say, already corroborated: the active server is the first one that answers *and* has the genesis of the network profile; every other configured server is asked, over a fresh connection with its own time limit, whether it agrees. Silence is never an answer (ELC-14): a server that does not respond does not count. """ import asyncio import hashlib import logging from collections.abc import Awaitable, Callable from embit.transaction import Transaction from enum import Enum from typing import NamedTuple from plm_engine.electrum.client import ElectrumClient, ElectrumEndpoint, ElectrumRpcError from plm_engine.electrum.merkle import proof_matches_header from plm_engine.electrum.scripthash import address_to_scripthash from plm_engine.headers import header_meets_its_own_target from plm_engine.network import Profile from plm_engine.psbt_builder import DUST_LIMIT_SATS from .deposits import Observation, Policy, ScanResult, Seen, confirmations, threshold_for logger = logging.getLogger(__name__) CONNECT_TIMEOUT = 10.0 ANSWER_TIMEOUT = 10.0 class Outcome(Enum): ACCEPTED = "accepted" REFUSED = "refused" UNCERTAIN = "uncertain" class Verdict(NamedTuple): outcome: Outcome reason: str # Wordings by which a node says it already has the transaction. Recognising one skips the verification; not # recognising one never releases anything, it only costs one more round of questions (BET-32, TX-04). _ALREADY_KNOWN = ("already known", "already in mempool", "already in the mempool", "txn-already-known", "txn-already-in-mempool", "already in block chain", "already in blockchain") def _seen(raw: dict) -> Seen | None: try: return Seen(txid=str(raw["tx_hash"]), vout=int(raw["tx_pos"]), value=int(raw["value"]), height=int(raw["height"])) except (KeyError, TypeError, ValueError): return None class ElectrumChain: def __init__( self, endpoints: list[ElectrumEndpoint], profile: Profile, allow_no_quorum: bool = False, client_factory: Callable[[ElectrumEndpoint], ElectrumClient] | None = None, ): self.endpoints = endpoints self.profile = profile self.allow_no_quorum = allow_no_quorum self._factory = client_factory or ( lambda e: ElectrumClient(e.host, e.port, e.use_ssl, e.pin, genesis_hash=profile.genesis_hash) ) self.active_endpoint: ElectrumEndpoint | None = None # -- connections ------------------------------------------------------------------------------ async def _open(self, endpoint: ElectrumEndpoint) -> ElectrumClient | None: client = self._factory(endpoint) try: await asyncio.wait_for(client.connect(), CONNECT_TIMEOUT) except Exception as error: # unreachable, wrong chain, bad pin: all are "this server is not usable now" logger.warning("electrum %s unusable: %s", endpoint, error) return None return client async def _connect_active(self) -> ElectrumClient | None: for endpoint in self.endpoints: # rotation: the first usable one in configuration order client = await self._open(endpoint) if client is not None: self.active_endpoint = endpoint return client return None async def _ask_others(self, ask: Callable[[ElectrumClient], Awaitable]) -> list: """`ask` on every server but the active one, concurrently, each over a fresh connection. `None` = no answer.""" async def one(endpoint: ElectrumEndpoint): client = await self._open(endpoint) if client is None: return None try: return await asyncio.wait_for(ask(client), ANSWER_TIMEOUT) except Exception as error: logger.warning("electrum %s did not answer: %s", endpoint, error) return None finally: await client.close() others = [e for e in self.endpoints if e != self.active_endpoint] return list(await asyncio.gather(*(one(e) for e in others))) def majority(self, answers: list) -> bool: """ELC-14. Answers are True (agrees), False (disagrees) or None (silent). At least two answered and a strict majority of those agrees; with the development exception, no other server at all passes and any strict majority of even one answer does.""" answered = [a for a in answers if a is not None] if self.allow_no_quorum: if not self.endpoints or len(self.endpoints) == 1: return True return bool(answered) and sum(bool(a) for a in answered) * 2 > len(answered) return len(answered) >= 2 and sum(bool(a) for a in answered) * 2 > len(answered) # -- what is asked ------------------------------------------------------------------------------ async def _tip(self, active: ElectrumClient) -> tuple[int, str] | None: """The active server's tip, accepted only with work valid for the profile and agreed by the others.""" reply = await active.subscribe_headers() try: height, header = int(reply["height"]), str(reply["hex"]) except (KeyError, TypeError, ValueError): return None if not header_meets_its_own_target(header, self.profile.pow_limit): logger.warning("the tip header of %s has no valid work", self.active_endpoint) return None async def same_header(client: ElectrumClient) -> bool: return await client.get_block_header(height) == header if not self.majority(await self._ask_others(same_header)): return None return height, header async def _unspent(self, client: ElectrumClient, address: str) -> list[Seen] | None: try: raw = await asyncio.wait_for(client.listunspent(address_to_scripthash(address)), ANSWER_TIMEOUT) except Exception as error: logger.warning("listunspent for %s failed: %s", address, error) return None if not isinstance(raw, list): return None seen = [_seen(item) for item in raw if isinstance(item, dict)] return None if any(item is None for item in seen) else seen async def is_coinbase(self, client: ElectrumClient, txid: str) -> bool | None: """From the raw transaction, accepted only if its hash is the txid (DEP-21); None = could not tell.""" try: raw = await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT) tx = Transaction.parse(bytes.fromhex(str(raw))) if tx.txid().hex() != txid: return None except Exception as error: logger.warning("cannot read transaction %s: %s", txid, error) return None return len(tx.vin) == 1 and tx.vin[0].txid == bytes(32) async def _credit_verified(self, active: ElectrumClient, address: str, entry: Seen, tip: int, need: int) -> bool: """ELC-17 (the others list the coin, same value, deep enough) and ELC-26 (a merkle proof from the others matches a header of the active server that has valid work).""" scripthash = address_to_scripthash(address) deepest = tip - need + 1 async def lists_it(client: ElectrumClient) -> bool: for raw in await client.listunspent(scripthash): other = _seen(raw) if isinstance(raw, dict) else None if other and (other.txid, other.vout) == (entry.txid, entry.vout): return other.value == entry.value and 0 < other.height <= deepest return False if not self.majority(await self._ask_others(lists_it)): return False return await self._proof_holds(active, entry.txid, entry.height) async def _proof_holds(self, active: ElectrumClient, txid: str, height: int) -> bool: """ELC-26, TX-22: a header of the active server at `height` with valid work for the profile, and a merkle proof from the others (by majority) that puts `txid` in that block.""" try: header = await asyncio.wait_for(active.get_block_header(height), ANSWER_TIMEOUT) except Exception: return False if not header_meets_its_own_target(str(header), self.profile.pow_limit): return False async def proves_it(client: ElectrumClient) -> bool: return proof_matches_header(txid, height, await client.get_merkle(txid, height), str(header)) return self.majority(await self._ask_others(proves_it)) async def _absent_confirmed(self, address: str, outpoint: tuple[str, int]) -> bool: scripthash = address_to_scripthash(address) async def lacks_it(client: ElectrumClient) -> bool: listed = {(r.get("tx_hash"), r.get("tx_pos")) for r in await client.listunspent(scripthash) if isinstance(r, dict)} return outpoint not in listed return self.majority(await self._ask_others(lacks_it)) # -- confirmations and presence (TX-30, TX-33) ---------------------------------------------------------------- @staticmethod async def _history(client: ElectrumClient, address: str) -> list[tuple[str, int]] | None: try: raw = await asyncio.wait_for(client.get_history(address_to_scripthash(address)), ANSWER_TIMEOUT) except Exception as error: logger.warning("history of %s failed: %s", address, error) return None try: return [(str(item["tx_hash"]), int(item["height"])) for item in raw] except (KeyError, TypeError, ValueError): return None async def observe_confirmations(self, wanted: list[tuple[int, str, frozenset[str]]]) -> dict[int, tuple[str, int]]: """`wanted` is `(row id, spending address, every txid of the row)`. Returns, per row, the version that is in a block *and* proven (TX-30): `{row id: (txid, height)}`. An address that cannot be read blocks nobody else.""" active = await self._connect_active() if active is None: return {} try: histories: dict[str, list[tuple[str, int]] | None] = {} confirmed: dict[int, tuple[str, int]] = {} for row_id, address, txids in wanted: if address not in histories: histories[address] = await self._history(active, address) history = histories[address] if history is None: continue for txid, height in history: if txid in txids and height > 0 and await self._proof_holds(active, txid, height): confirmed[row_id] = (txid, height) break return confirmed except Exception: logger.exception("confirmation poll failed") return {} finally: await active.close() async def versions_known(self, address: str, txids: frozenset[str]) -> tuple[set[str], bool]: """TX-33: which of `txids` any server of the network has seen (mempool or block), and whether the network answered enough for the *absence* of the rest to mean something (the active server and the quorum).""" active = await self._connect_active() if active is None: return set(), False try: mine = await self._history(active, address) async def theirs(client: ElectrumClient): history = await self._history(client, address) return None if history is None else {txid for txid, _ in history if txid in txids} others = await self._ask_others(theirs) known = ({txid for txid, _ in mine if txid in txids} if mine is not None else set()) for answer in others: if answer: known |= answer # an absence is a statement only if the active server answered and the others agree among themselves complete = mine is not None and self.majority([a is not None for a in others]) return known, complete finally: await active.close() # -- broadcast (BET-32) ----------------------------------------------------------------------------------- async def transaction_known(self, active: ElectrumClient, txid: str) -> bool | None: """True if any server returns the transaction; False only if the active server and the quorum of the others all say they do not know it; None if there were not enough answers to say.""" not_found = ("no such mempool", "no such blockchain", "not found", "unknown transaction") async def ask(client: ElectrumClient): try: await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT) return True except ElectrumRpcError as error: # the server answered: it does not know it return False if any(m in error.message.lower() for m in not_found) else None except Exception: return None mine = await ask(active) others = await self._ask_others(ask) if mine is True or any(a is True for a in others): return True if mine is False and self.majority([a is False if a is not None else None for a in others]): return False return None async def broadcast(self, raw_tx_hex: str, txid: str) -> Verdict: """Send `raw_tx_hex` through the active server and say what happened (BET-32). Only a proven refusal frees coins.""" active = await self._connect_active() if active is None: return Verdict(Outcome.UNCERTAIN, "no Electrum server is reachable") try: try: await asyncio.wait_for(active.broadcast(raw_tx_hex), ANSWER_TIMEOUT) except ElectrumRpcError as error: reason = error.message known = True if any(m in reason.lower() for m in _ALREADY_KNOWN) else await self.transaction_known(active, txid) if known is True: await self._fan_out(raw_tx_hex) return Verdict(Outcome.ACCEPTED, reason) if known is False: return Verdict(Outcome.REFUSED, reason) logger.warning("broadcast of %s was refused (%s) but the refusal could not be verified", txid, reason) return Verdict(Outcome.UNCERTAIN, reason) except Exception as error: # timeout, dropped connection: the server may have relayed it logger.warning("broadcast of %s ended without an answer (%r): uncertain", txid, error) return Verdict(Outcome.UNCERTAIN, str(error) or type(error).__name__) await self._fan_out(raw_tx_hex) return Verdict(Outcome.ACCEPTED, "") finally: await active.close() async def _fan_out(self, raw_tx_hex: str) -> None: """TX-21: after the active server accepted, tell the others too (best effort).""" async def send(client: ElectrumClient): try: await client.broadcast(raw_tx_hex) except ElectrumRpcError: pass # "already known" and the like: that is what we wanted return True try: await self._ask_others(send) except Exception: logger.exception("fan-out of an accepted transaction failed") # -- the scan ---------------------------------------------------------------------------------------- async def observe(self, policy: Policy) -> ScanResult: active = await self._connect_active() if active is None: return ScanResult(None, {}) try: tip = await self._tip(active) if tip is None: return ScanResult(None, {}) result = ScanResult(tip[0], {}) for address in policy.addresses: result.addresses[address] = await self._observe_address(active, address, tip[0], policy) return result except Exception: # a scan that fails writes nothing; the next one starts again logger.exception("deposit scan failed") return ScanResult(None, {}) finally: await active.close() async def _observe_address(self, active: ElectrumClient, address: str, tip: int, policy: Policy) -> Observation: entries = await self._unspent(active, address) observation = Observation(entries) if entries is None: return observation listed = {(e.txid, e.vout) for e in entries} for entry in entries: outpoint = (entry.txid, entry.vout) if entry.value < DUST_LIMIT_SATS or entry.height <= 0 or outpoint in policy.credited: continue have = confirmations(tip, entry.height) if have >= policy.deposit_confirmations and entry.txid not in policy.own_txids: coinbase = await self.is_coinbase(active, entry.txid) if coinbase is None: continue # unreadable: it cannot be credited as an ordinary deposit by mistake if coinbase: observation.coinbase.add(entry.txid) need = threshold_for(entry.txid, policy, observation.coinbase) if have >= need and await self._credit_verified(active, address, entry, tip, need): observation.verified.add(outpoint) for outpoint in policy.waiting.get(address, frozenset()) - listed: if await self._absent_confirmed(address, outpoint): observation.absent_confirmed.add(outpoint) return observation