310 lines
14 KiB
Python
310 lines
14 KiB
Python
"""The network phase of a deposit scan (roadmap B5, DEP-20, ELC-14, ELC-17, ELC-26, ELC-31).
|
|
|
|
Nothing here touches the database. `observe` takes the `Policy` read in phase 1 and returns what the servers
|
|
say, already corroborated: the active server is the first one that answers *and* has the genesis of the network
|
|
profile; every other configured server is asked, over a fresh connection with its own time limit, whether it
|
|
agrees. Silence is never an answer (ELC-14): a server that does not respond does not count.
|
|
"""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
from collections.abc import Awaitable, Callable
|
|
|
|
from embit.transaction import Transaction
|
|
from enum import Enum
|
|
from typing import NamedTuple
|
|
|
|
from plm_engine.electrum.client import ElectrumClient, ElectrumEndpoint, ElectrumRpcError
|
|
from plm_engine.electrum.merkle import proof_matches_header
|
|
from plm_engine.electrum.scripthash import address_to_scripthash
|
|
from plm_engine.headers import header_meets_its_own_target
|
|
from plm_engine.network import Profile
|
|
from plm_engine.psbt_builder import DUST_LIMIT_SATS
|
|
|
|
from .deposits import Observation, Policy, ScanResult, Seen, confirmations, threshold_for
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
CONNECT_TIMEOUT = 10.0
|
|
ANSWER_TIMEOUT = 10.0
|
|
|
|
|
|
class Outcome(Enum):
|
|
ACCEPTED = "accepted"
|
|
REFUSED = "refused"
|
|
UNCERTAIN = "uncertain"
|
|
|
|
|
|
class Verdict(NamedTuple):
|
|
outcome: Outcome
|
|
reason: str
|
|
|
|
|
|
# Wordings by which a node says it already has the transaction. Recognising one skips the verification; not
|
|
# recognising one never releases anything, it only costs one more round of questions (BET-32, TX-04).
|
|
_ALREADY_KNOWN = ("already known", "already in mempool", "already in the mempool", "txn-already-known",
|
|
"txn-already-in-mempool", "already in block chain", "already in blockchain")
|
|
|
|
|
|
def _seen(raw: dict) -> Seen | None:
|
|
try:
|
|
return Seen(txid=str(raw["tx_hash"]), vout=int(raw["tx_pos"]), value=int(raw["value"]), height=int(raw["height"]))
|
|
except (KeyError, TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
class ElectrumChain:
|
|
def __init__(
|
|
self,
|
|
endpoints: list[ElectrumEndpoint],
|
|
profile: Profile,
|
|
allow_no_quorum: bool = False,
|
|
client_factory: Callable[[ElectrumEndpoint], ElectrumClient] | None = None,
|
|
):
|
|
self.endpoints = endpoints
|
|
self.profile = profile
|
|
self.allow_no_quorum = allow_no_quorum
|
|
self._factory = client_factory or (
|
|
lambda e: ElectrumClient(e.host, e.port, e.use_ssl, e.pin, genesis_hash=profile.genesis_hash)
|
|
)
|
|
self.active_endpoint: ElectrumEndpoint | None = None
|
|
|
|
# -- connections ------------------------------------------------------------------------------
|
|
|
|
async def _open(self, endpoint: ElectrumEndpoint) -> ElectrumClient | None:
|
|
client = self._factory(endpoint)
|
|
try:
|
|
await asyncio.wait_for(client.connect(), CONNECT_TIMEOUT)
|
|
except Exception as error: # unreachable, wrong chain, bad pin: all are "this server is not usable now"
|
|
logger.warning("electrum %s unusable: %s", endpoint, error)
|
|
return None
|
|
return client
|
|
|
|
async def _connect_active(self) -> ElectrumClient | None:
|
|
for endpoint in self.endpoints: # rotation: the first usable one in configuration order
|
|
client = await self._open(endpoint)
|
|
if client is not None:
|
|
self.active_endpoint = endpoint
|
|
return client
|
|
return None
|
|
|
|
async def _ask_others(self, ask: Callable[[ElectrumClient], Awaitable]) -> list:
|
|
"""`ask` on every server but the active one, concurrently, each over a fresh connection. `None` = no answer."""
|
|
|
|
async def one(endpoint: ElectrumEndpoint):
|
|
client = await self._open(endpoint)
|
|
if client is None:
|
|
return None
|
|
try:
|
|
return await asyncio.wait_for(ask(client), ANSWER_TIMEOUT)
|
|
except Exception as error:
|
|
logger.warning("electrum %s did not answer: %s", endpoint, error)
|
|
return None
|
|
finally:
|
|
await client.close()
|
|
|
|
others = [e for e in self.endpoints if e != self.active_endpoint]
|
|
return list(await asyncio.gather(*(one(e) for e in others)))
|
|
|
|
def majority(self, answers: list) -> bool:
|
|
"""ELC-14. Answers are True (agrees), False (disagrees) or None (silent). At least two answered and a strict
|
|
majority of those agrees; with the development exception, no other server at all passes and any strict majority
|
|
of even one answer does."""
|
|
answered = [a for a in answers if a is not None]
|
|
if self.allow_no_quorum:
|
|
if not self.endpoints or len(self.endpoints) == 1:
|
|
return True
|
|
return bool(answered) and sum(bool(a) for a in answered) * 2 > len(answered)
|
|
return len(answered) >= 2 and sum(bool(a) for a in answered) * 2 > len(answered)
|
|
|
|
# -- what is asked ------------------------------------------------------------------------------
|
|
|
|
async def _tip(self, active: ElectrumClient) -> tuple[int, str] | None:
|
|
"""The active server's tip, accepted only with work valid for the profile and agreed by the others."""
|
|
reply = await active.subscribe_headers()
|
|
try:
|
|
height, header = int(reply["height"]), str(reply["hex"])
|
|
except (KeyError, TypeError, ValueError):
|
|
return None
|
|
if not header_meets_its_own_target(header, self.profile.pow_limit):
|
|
logger.warning("the tip header of %s has no valid work", self.active_endpoint)
|
|
return None
|
|
|
|
async def same_header(client: ElectrumClient) -> bool:
|
|
return await client.get_block_header(height) == header
|
|
|
|
if not self.majority(await self._ask_others(same_header)):
|
|
return None
|
|
return height, header
|
|
|
|
async def _unspent(self, client: ElectrumClient, address: str) -> list[Seen] | None:
|
|
try:
|
|
raw = await asyncio.wait_for(client.listunspent(address_to_scripthash(address)), ANSWER_TIMEOUT)
|
|
except Exception as error:
|
|
logger.warning("listunspent for %s failed: %s", address, error)
|
|
return None
|
|
if not isinstance(raw, list):
|
|
return None
|
|
seen = [_seen(item) for item in raw if isinstance(item, dict)]
|
|
return None if any(item is None for item in seen) else seen
|
|
|
|
async def is_coinbase(self, client: ElectrumClient, txid: str) -> bool | None:
|
|
"""From the raw transaction, accepted only if its hash is the txid (DEP-21); None = could not tell."""
|
|
try:
|
|
raw = await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT)
|
|
tx = Transaction.parse(bytes.fromhex(str(raw)))
|
|
if tx.txid().hex() != txid:
|
|
return None
|
|
except Exception as error:
|
|
logger.warning("cannot read transaction %s: %s", txid, error)
|
|
return None
|
|
return len(tx.vin) == 1 and tx.vin[0].txid == bytes(32)
|
|
|
|
async def _credit_verified(self, active: ElectrumClient, address: str, entry: Seen, tip: int, need: int) -> bool:
|
|
"""ELC-17 (the others list the coin, same value, deep enough) and ELC-26 (a merkle proof from the others
|
|
matches a header of the active server that has valid work)."""
|
|
scripthash = address_to_scripthash(address)
|
|
deepest = tip - need + 1
|
|
|
|
async def lists_it(client: ElectrumClient) -> bool:
|
|
for raw in await client.listunspent(scripthash):
|
|
other = _seen(raw) if isinstance(raw, dict) else None
|
|
if other and (other.txid, other.vout) == (entry.txid, entry.vout):
|
|
return other.value == entry.value and 0 < other.height <= deepest
|
|
return False
|
|
|
|
if not self.majority(await self._ask_others(lists_it)):
|
|
return False
|
|
try:
|
|
header = await asyncio.wait_for(active.get_block_header(entry.height), ANSWER_TIMEOUT)
|
|
except Exception:
|
|
return False
|
|
if not header_meets_its_own_target(str(header), self.profile.pow_limit):
|
|
return False
|
|
|
|
async def proves_it(client: ElectrumClient) -> bool:
|
|
return proof_matches_header(entry.txid, entry.height, await client.get_merkle(entry.txid, entry.height), str(header))
|
|
|
|
return self.majority(await self._ask_others(proves_it))
|
|
|
|
async def _absent_confirmed(self, address: str, outpoint: tuple[str, int]) -> bool:
|
|
scripthash = address_to_scripthash(address)
|
|
|
|
async def lacks_it(client: ElectrumClient) -> bool:
|
|
listed = {(r.get("tx_hash"), r.get("tx_pos")) for r in await client.listunspent(scripthash) if isinstance(r, dict)}
|
|
return outpoint not in listed
|
|
|
|
return self.majority(await self._ask_others(lacks_it))
|
|
|
|
# -- broadcast (BET-32) -----------------------------------------------------------------------------------
|
|
|
|
async def transaction_known(self, active: ElectrumClient, txid: str) -> bool | None:
|
|
"""True if any server returns the transaction; False only if the active server and the quorum of the
|
|
others all say they do not know it; None if there were not enough answers to say."""
|
|
not_found = ("no such mempool", "no such blockchain", "not found", "unknown transaction")
|
|
|
|
async def ask(client: ElectrumClient):
|
|
try:
|
|
await asyncio.wait_for(client.get_transaction(txid), ANSWER_TIMEOUT)
|
|
return True
|
|
except ElectrumRpcError as error: # the server answered: it does not know it
|
|
return False if any(m in error.message.lower() for m in not_found) else None
|
|
except Exception:
|
|
return None
|
|
|
|
mine = await ask(active)
|
|
others = await self._ask_others(ask)
|
|
if mine is True or any(a is True for a in others):
|
|
return True
|
|
if mine is False and self.majority([a is False if a is not None else None for a in others]):
|
|
return False
|
|
return None
|
|
|
|
async def broadcast(self, raw_tx_hex: str, txid: str) -> Verdict:
|
|
"""Send `raw_tx_hex` through the active server and say what happened (BET-32). Only a proven refusal frees coins."""
|
|
active = await self._connect_active()
|
|
if active is None:
|
|
return Verdict(Outcome.UNCERTAIN, "no Electrum server is reachable")
|
|
try:
|
|
try:
|
|
await asyncio.wait_for(active.broadcast(raw_tx_hex), ANSWER_TIMEOUT)
|
|
except ElectrumRpcError as error:
|
|
reason = error.message
|
|
known = True if any(m in reason.lower() for m in _ALREADY_KNOWN) else await self.transaction_known(active, txid)
|
|
if known is True:
|
|
await self._fan_out(raw_tx_hex)
|
|
return Verdict(Outcome.ACCEPTED, reason)
|
|
if known is False:
|
|
return Verdict(Outcome.REFUSED, reason)
|
|
logger.warning("broadcast of %s was refused (%s) but the refusal could not be verified", txid, reason)
|
|
return Verdict(Outcome.UNCERTAIN, reason)
|
|
except Exception as error: # timeout, dropped connection: the server may have relayed it
|
|
logger.warning("broadcast of %s ended without an answer (%r): uncertain", txid, error)
|
|
return Verdict(Outcome.UNCERTAIN, str(error) or type(error).__name__)
|
|
await self._fan_out(raw_tx_hex)
|
|
return Verdict(Outcome.ACCEPTED, "")
|
|
finally:
|
|
await active.close()
|
|
|
|
async def _fan_out(self, raw_tx_hex: str) -> None:
|
|
"""TX-21: after the active server accepted, tell the others too (best effort)."""
|
|
|
|
async def send(client: ElectrumClient):
|
|
try:
|
|
await client.broadcast(raw_tx_hex)
|
|
except ElectrumRpcError:
|
|
pass # "already known" and the like: that is what we wanted
|
|
return True
|
|
|
|
try:
|
|
await self._ask_others(send)
|
|
except Exception:
|
|
logger.exception("fan-out of an accepted transaction failed")
|
|
|
|
# -- the scan ----------------------------------------------------------------------------------------
|
|
|
|
async def observe(self, policy: Policy) -> ScanResult:
|
|
active = await self._connect_active()
|
|
if active is None:
|
|
return ScanResult(None, {})
|
|
try:
|
|
tip = await self._tip(active)
|
|
if tip is None:
|
|
return ScanResult(None, {})
|
|
result = ScanResult(tip[0], {})
|
|
for address in policy.addresses:
|
|
result.addresses[address] = await self._observe_address(active, address, tip[0], policy)
|
|
return result
|
|
except Exception: # a scan that fails writes nothing; the next one starts again
|
|
logger.exception("deposit scan failed")
|
|
return ScanResult(None, {})
|
|
finally:
|
|
await active.close()
|
|
|
|
async def _observe_address(self, active: ElectrumClient, address: str, tip: int, policy: Policy) -> Observation:
|
|
entries = await self._unspent(active, address)
|
|
observation = Observation(entries)
|
|
if entries is None:
|
|
return observation
|
|
listed = {(e.txid, e.vout) for e in entries}
|
|
for entry in entries:
|
|
outpoint = (entry.txid, entry.vout)
|
|
if entry.value < DUST_LIMIT_SATS or entry.height <= 0 or outpoint in policy.credited:
|
|
continue
|
|
have = confirmations(tip, entry.height)
|
|
if have >= policy.deposit_confirmations and entry.txid not in policy.own_txids:
|
|
coinbase = await self.is_coinbase(active, entry.txid)
|
|
if coinbase is None:
|
|
continue # unreadable: it cannot be credited as an ordinary deposit by mistake
|
|
if coinbase:
|
|
observation.coinbase.add(entry.txid)
|
|
need = threshold_for(entry.txid, policy, observation.coinbase)
|
|
if have >= need and await self._credit_verified(active, address, entry, tip, need):
|
|
observation.verified.add(outpoint)
|
|
for outpoint in policy.waiting.get(address, frozenset()) - listed:
|
|
if await self._absent_confirmed(address, outpoint):
|
|
observation.absent_confirmed.add(outpoint)
|
|
return observation
|
|
|