"""B-78 (TX-14, TX-15, TX-17, TX-23, TX-26, ELC-28, DAT-33, DAT-36, DAT-69, DAT-72, AUD-11, AUD-12, AUD-31, AUD-39, API-32): a row is abandoned only on proof, and failed rows stay watched. The old rule gave a row up when the *active* server's history lacked its txid — after 120s for `building`, 6h for `pending`. After a restart onto a different server, or a forward clock jump, that released the coins of transactions that were alive: a bet's sats sat in the pool with no participant, a withdrawal left while marked failed, a payout was rebuilt and paid twice. Each test here fails against that rule. """ from datetime import datetime, timedelta, timezone import pytest from embit import script from embit.transaction import Transaction, TransactionInput, TransactionOutput from sqlalchemy import select from app.db.models import AuditLog, PendingTransaction, PendingTransactionTxid, RoundConfig, RoundParticipant, User, UtxoEvent, Withdrawal from app.tx import reconcile as reconcile_module from app.tx.reconcile import FAILED_WATCH_DAYS, reconcile_once from tests.unit.fake_network import MISSING_INPUTS, ScriptedNetwork from tests.unit.test_reconcile import ( # noqa: F401 (fixture and helpers shared with the reconciler tests) _RAW_TX, _TX_INPUT_TXID, UnknownTxClient, _seed_bet, session_factory, ) _BET_TXID = "betxid" _KNOWN = [{"tx_hash": _BET_TXID, "height": 0}] async def _row(session_factory): async with session_factory() as session: return (await session.scalars(select(PendingTransaction))).one() async def _events(session_factory): async with session_factory() as session: return [e.event_type for e in (await session.scalars(select(AuditLog))).all()] async def _seed_building(session_factory, **kwargs): return await _seed_bet(session_factory, pending_status="building", participant_status="building", age_seconds=300, **kwargs) # --- presence: any server, any version (TX-15, TX-16) -------------------------------------- async def test_a_row_the_active_server_has_not_seen_is_promoted_when_another_server_knows_it(session_factory): """The restart-onto-another-server case: the new active server has no record of the tx, a different node holds it in its mempool. Abandoning here released a live tx.""" await _seed_building(session_factory) network = ScriptedNetwork(active=[], others=(_KNOWN, [])) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 row = await _row(session_factory) assert row.status == "pending" assert network.rebroadcast_calls == [] # nothing needed proving async with session_factory() as session: assert (await session.scalars(select(RoundParticipant))).one().status == "broadcast" assert (await session.scalars(select(UtxoEvent))).one().spent_txid == _BET_TXID assert "pending_tx_abandoned" not in await _events(session_factory) # --- silence is not proof (TX-15, TX-17) -------------------------------------------------- @pytest.mark.parametrize("age_seconds, status", [(300, "building"), (7 * 3600, "pending")]) async def test_too_few_answers_leave_the_row_alone(session_factory, age_seconds, status): """Only one of the other servers answered, below the quorum minimum: the empty histories prove nothing and nothing is even rebroadcast.""" await _seed_bet( session_factory, pending_status=status, participant_status="building" if status == "building" else "broadcast", age_seconds=age_seconds, ) network = ScriptedNetwork.proof_of_absence(active=[], others=([], None)) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 0 assert (await _row(session_factory)).status == status assert network.rebroadcast_calls == [] assert "pending_tx_abandoned" not in await _events(session_factory) async def test_an_active_server_that_does_not_answer_leaves_the_row_alone(session_factory): await _seed_building(session_factory) network = ScriptedNetwork.proof_of_absence(active=None, others=([], [])) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 0 assert (await _row(session_factory)).status == "building" async def test_no_network_never_abandons(session_factory): """With nobody to ask beyond the active server, silence cannot be told from absence.""" await _seed_building(session_factory) assert await reconcile_once(session_factory, UnknownTxClient()) == 0 assert (await _row(session_factory)).status == "building" async def test_a_rebroadcast_that_one_server_accepts_means_the_tx_is_alive(session_factory): """TX-17: nobody listed it, but one server takes it when it is sent again — it is alive (or alive again), never abandoned; a `building` row is promoted.""" await _seed_building(session_factory) network = ScriptedNetwork(rebroadcast=(("refused", MISSING_INPUTS), (("accepted", ""), ("refused", MISSING_INPUTS)))) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 assert (await _row(session_factory)).status == "pending" assert "pending_tx_abandoned" not in await _events(session_factory) async def test_an_already_known_refusal_of_the_rebroadcast_means_the_tx_is_alive(session_factory): await _seed_building(session_factory) network = ScriptedNetwork(rebroadcast=(("refused", "txn-already-in-mempool"), (None, None))) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 assert (await _row(session_factory)).status == "pending" async def test_a_fee_refusal_is_not_proof_a_building_row_goes_to_pending_for_the_bumper(session_factory): """TX-17: the rebroadcast was refused for its fee — the tx may well be valid at a higher one, so nothing is released; `building` becomes `pending` so RBF can bump it.""" await _seed_building(session_factory) fee = ("refused", "min relay fee not met") network = ScriptedNetwork(rebroadcast=(fee, (fee, fee))) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 assert (await _row(session_factory)).status == "pending" async with session_factory() as session: assert (await session.scalars(select(UtxoEvent))).one().spent_txid == _BET_TXID assert "pending_tx_abandoned" not in await _events(session_factory) async def test_a_fee_refusal_leaves_a_pending_row_as_it_is(session_factory): await _seed_bet(session_factory, pending_status="pending", participant_status="broadcast", age_seconds=7 * 3600) fee = ("refused", "insufficient fee") network = ScriptedNetwork(rebroadcast=(fee, (fee, fee))) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 0 assert (await _row(session_factory)).status == "pending" @pytest.mark.parametrize( "rebroadcast", [ pytest.param((("refused", "txn-mempool-conflict"), (("refused", "txn-mempool-conflict"),) * 2), id="mempool-conflict"), pytest.param((("refused", MISSING_INPUTS), (("refused", MISSING_INPUTS), None)), id="too-few-refusals"), pytest.param((("refused", MISSING_INPUTS), (("refused", MISSING_INPUTS), ("refused", "something else"))), id="mixed"), pytest.param((None, (None, None)), id="nobody-answers"), ], ) async def test_a_rebroadcast_that_is_not_a_unanimous_missing_inputs_refusal_is_not_proof(session_factory, rebroadcast): await _seed_building(session_factory) network = ScriptedNetwork(rebroadcast=rebroadcast) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 0 assert (await _row(session_factory)).status == "building" # --- the two proofs (TX-17) ------------------------------------------------------------- async def test_an_input_spent_in_a_block_by_another_transaction_is_proof(session_factory): await _seed_building(session_factory) spender = "ab" * 32 network = ScriptedNetwork(spender=spender) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 row = await _row(session_factory) assert row.status == "failed" and spender in row.failure_reason assert row.failed_at is not None async with session_factory() as session: assert (await session.scalars(select(UtxoEvent))).one().spent_txid is None assert (await session.scalars(select(RoundParticipant))).all() == [] abandoned = ( await session.scalars(select(AuditLog).where(AuditLog.event_type == "pending_tx_abandoned")) ).one() assert spender in abandoned.payload_json async def test_a_unanimous_missing_inputs_refusal_of_the_rebroadcast_is_proof(session_factory): await _seed_bet(session_factory, pending_status="pending", participant_status="broadcast", age_seconds=7 * 3600) network = ScriptedNetwork.proof_of_absence() assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 row = await _row(session_factory) assert row.status == "failed" and "missing" in row.failure_reason and row.failed_at is not None assert network.rebroadcast_calls == [_RAW_TX] # the latest raw tx, to every server async with session_factory() as session: assert (await session.scalars(select(UtxoEvent))).one().spent_txid is None # --- age alone proves nothing (TX-14) ----------------------------------------------------- async def test_a_forward_clock_jump_abandons_nothing(session_factory, monkeypatch): """Seven hours jump by on the host's clock: every `pending` row is suddenly past the 6h grace period. They are examined — and, the network knowing them, left alone.""" await _seed_bet(session_factory, pending_status="pending", participant_status="broadcast", age_seconds=60) real_datetime = reconcile_module.datetime class _Jumped(real_datetime): @classmethod def now(cls, tz=None): return real_datetime.now(tz) + timedelta(hours=7) monkeypatch.setattr(reconcile_module, "datetime", _Jumped) # The row is examined (a real network that does not know the tx would have to prove # its absence) — here one server lists it. network = ScriptedNetwork(active=[], others=(_KNOWN, [])) await reconcile_once(session_factory, UnknownTxClient(), network) assert network.history_calls == 1 # it was examined assert (await _row(session_factory)).status == "pending" # And where nobody lists it but the network is only silent, still nothing is given up. quiet = ScriptedNetwork(active=[], others=([], None)) await reconcile_once(session_factory, UnknownTxClient(), quiet) assert (await _row(session_factory)).status == "pending" assert "pending_tx_abandoned" not in await _events(session_factory) # --- the re-check (TX-26) --------------------------------------------------------------------- async def test_a_row_bumped_while_the_network_was_asked_is_not_abandoned(session_factory): """The answer was about the txid the row had when it was asked; an RBF that lands in the meantime changes what it means, so the outcome is not written.""" await _seed_bet(session_factory, pending_status="pending", participant_status="broadcast", age_seconds=7 * 3600) class BumpingNetwork(ScriptedNetwork): async def confirmed_spender(self, *args): async with session_factory() as session: row = (await session.scalars(select(PendingTransaction))).one() row.current_txid = "bumped-txid" await session.commit() return "ab" * 32 # would be proof — about the old txid assert await reconcile_once(session_factory, UnknownTxClient(), BumpingNetwork()) == 0 row = await _row(session_factory) assert row.status == "pending" and row.current_txid == "bumped-txid" # --- failed rows stay watched (TX-23, DAT-72) --------------------------------------------------- async def _seed_failed_withdrawal(session_factory, *, failed_days_ago: float = 1.0, with_earlier_version: str | None = None): from app.wallet.hd import derive_user_address failed_at = datetime.now(timezone.utc) - timedelta(days=failed_days_ago) async with session_factory() as session: user = User(username="w", password_hash="x", derivation_index=1, address=derive_user_address(1)) session.add(user) await session.flush() session.add(UtxoEvent(user_id=user.id, txid=_TX_INPUT_TXID, vout=0, amount_sats=500_000_000, confirmed_height=10)) withdrawal = Withdrawal( user_id=user.id, external_address="plm1qexternal", amount_requested_sats=400_000_000, amount_sent_sats=399_000_000, txid=None, status="failed", ) session.add(withdrawal) await session.flush() pending = PendingTransaction( kind="withdrawal", withdrawal_id=withdrawal.id, user_id=user.id, current_txid="wdtxid", fee_rate_sat_vb=1, raw_tx_hex=_RAW_TX, status="failed", failure_reason="rebroadcast refused for missing inputs", failed_at=failed_at, broadcast_at=failed_at - timedelta(days=1), ) session.add(pending) await session.flush() session.add(PendingTransactionTxid(pending_transaction_id=pending.id, txid="wdtxid", raw_hex=_RAW_TX)) if with_earlier_version: session.add(PendingTransactionTxid(pending_transaction_id=pending.id, txid=with_earlier_version, raw_hex=_RAW_TX)) await session.commit() async def test_a_failed_row_that_confirms_later_is_resurrected_and_pauses_the_lottery(session_factory): await _seed_failed_withdrawal(session_factory) mined = [{"tx_hash": "wdtxid", "height": 321}] network = ScriptedNetwork(active=mined, others=(mined, mined)) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 async with session_factory() as session: row = (await session.scalars(select(PendingTransaction))).one() assert row.status == "confirmed" config = (await session.scalars(select(RoundConfig))).one() assert config.paused is True and config.paused_reason == "resurrected_tx" resurrected = (await session.scalars(select(AuditLog).where(AuditLog.event_type == "pending_tx_resurrected"))).one() assert '"block_height": 321' in resurrected.payload_json and '"txid": "wdtxid"' in resurrected.payload_json assert resurrected.user_id == row.user_id paused = (await session.scalars(select(AuditLog).where(AuditLog.event_type == "lottery_paused"))).one() assert '"reason": "resurrected_tx"' in paused.payload_json # The linked rows are not corrected by the system (DAT-72): the withdrawal stays failed. assert (await session.scalars(select(Withdrawal))).one().status == "failed" assert (await session.scalars(select(UtxoEvent))).one().spent_txid is None # Written once: the row is confirmed now and no longer watched. assert await reconcile_once(session_factory, UnknownTxClient(), network) == 0 events = await _events(session_factory) assert events.count("pending_tx_resurrected") == 1 and events.count("lottery_paused") == 1 async def test_an_earlier_version_of_a_failed_row_that_confirms_is_resurrected_too(session_factory): await _seed_failed_withdrawal(session_factory, with_earlier_version="older-txid") mined = [{"tx_hash": "older-txid", "height": 400}] network = ScriptedNetwork(active=[], others=(mined, mined)) assert await reconcile_once(session_factory, UnknownTxClient(), network) == 1 row = await _row(session_factory) assert row.status == "confirmed" and row.current_txid == "older-txid" async def test_a_failed_row_the_quorum_does_not_confirm_stays_failed(session_factory): """Only the active server (or only a minority of the others) says it is mined.""" await _seed_failed_withdrawal(session_factory) mined = [{"tx_hash": "wdtxid", "height": 321}] hostile_active = ScriptedNetwork(active=mined, others=([], [])) assert await reconcile_once(session_factory, UnknownTxClient(), hostile_active) == 0 minority = ScriptedNetwork(active=mined, others=(mined, [], []), others_needed=2) assert await reconcile_once(session_factory, UnknownTxClient(), minority) == 0 too_few = ScriptedNetwork(active=mined, others=(mined, None), others_needed=2) assert await reconcile_once(session_factory, UnknownTxClient(), too_few) == 0 assert (await _row(session_factory)).status == "failed" async with session_factory() as session: config = (await session.scalars(select(RoundConfig))).first() assert config is None or not config.paused # no pause without the quorum async def test_a_failed_row_is_watched_for_fourteen_days_and_no_longer(session_factory): await _seed_failed_withdrawal(session_factory, failed_days_ago=FAILED_WATCH_DAYS - 1) network = ScriptedNetwork() await reconcile_once(session_factory, UnknownTxClient(), network) assert network.history_calls == 1 # still watched async with session_factory() as session: row = (await session.scalars(select(PendingTransaction))).one() row.failed_at = datetime.now(timezone.utc) - timedelta(days=FAILED_WATCH_DAYS + 1) await session.commit() network = ScriptedNetwork() await reconcile_once(session_factory, UnknownTxClient(), network) assert network.history_calls == 0 # past the watch async def test_a_refused_withdrawal_starts_its_watch_too(session_factory): """DAT-36: `_release_failed_withdrawal` stamps failed_at like `_abandon`.""" from app.electrum.client import ElectrumRpcError from app.withdrawals.service import WithdrawalError, request_withdrawal from tests.unit.fake_network import FakeNetwork from app.wallet.hd import derive_user_address async with session_factory() as session: user = User(username="r", password_hash="x", derivation_index=2, address=derive_user_address(2)) session.add(user) await session.flush() session.add(UtxoEvent(user_id=user.id, txid="22" * 32, vout=0, amount_sats=3_000_000_000, confirmed_height=10)) session.add(RoundConfig(fee_address="plm1q5x25wd6q463mfhckjraaedgjg0lyu73qfcj43n")) await session.commit() user_id = user.id class Refusing: async def broadcast(self, raw): raise ElectrumRpcError({"code": -26, "message": "min relay fee not met"}) async with session_factory() as session: user = await session.get(User, user_id) with pytest.raises(WithdrawalError): await request_withdrawal( session, Refusing(), user, "plm1qqph9qup2mp7w7g5nlsdhdc9m2pp44ampzw0ctx", 1_000_000_000, network=FakeNetwork(known=False), ) row = await _row(session_factory) assert row.status == "failed" and row.failed_at is not None # --- the listener's side: asking every server (TX-15, TX-17, ELC-28) --------------------------- class _Server: """One fake connection. `history`: the address's entries (None = unreachable); `txs`: raw transactions by txid it can return; `broadcast`: "ok", an error message (an explicit JSON-RPC refusal) or None (no answer).""" def __init__(self, history=(), txs=None, broadcast="ok"): self._history = history self._txs = txs or {} self._broadcast = broadcast self.broadcasted: list[str] = [] async def connect(self): if self._history is None: raise ConnectionRefusedError("down") async def get_history(self, scripthash): return list(self._history) async def get_transaction(self, txid, verbose=False): return self._txs[txid] async def broadcast(self, raw): from app.electrum.client import ElectrumError, ElectrumRpcError self.broadcasted.append(raw) if self._broadcast is None: raise ElectrumError("timed out") if self._broadcast != "ok": raise ElectrumRpcError({"code": -25, "message": self._broadcast}) return "txid" async def close(self): pass def _make_listener(session_factory, active: _Server | None, others: dict[str, _Server]): from app.electrum.client import ElectrumEndpoint from app.electrum.listener import ElectrumListener endpoints = [ElectrumEndpoint("active.example", 50002, True)] + [ElectrumEndpoint(h, 50002, True) for h in others] listener = ElectrumListener(lambda endpoint: others[endpoint.host], session_factory, endpoints) listener.client = active return listener def _spending_tx(spent_txid: str, vout: int = 0): """A real transaction spending (spent_txid, vout), and its txid.""" tx = Transaction( vin=[TransactionInput(bytes.fromhex(spent_txid), vout)], vout=[TransactionOutput(1_000, script.Script.from_address("plm1q0xcqpzrky6eff2g52qdye53xkk9jxkvraxkkwd"))], ) return tx.serialize().hex(), tx.txid().hex() async def test_get_histories_returns_every_servers_own_answer(session_factory): listener = _make_listener( session_factory, _Server([{"tx_hash": "aa", "height": 1}]), {"first.example": _Server([]), "second.example": _Server(None), "third.example": _Server([{"tx_hash": "bb", "height": 0}])}, ) answers = await listener.get_histories("scripthash") assert answers.active == [{"tx_hash": "aa", "height": 1}] assert answers.others == ([], None, [{"tx_hash": "bb", "height": 0}]) assert answers.complete # the active server and two of the three others answered async def test_get_histories_is_incomplete_without_the_active_server_or_enough_others(session_factory): no_active = _make_listener(session_factory, None, {"first.example": _Server([]), "second.example": _Server([])}) assert not (await no_active.get_histories("s")).complete too_few = _make_listener(session_factory, _Server([]), {"first.example": _Server([]), "second.example": _Server(None)}) assert not (await too_few.get_histories("s")).complete async def test_rebroadcast_everywhere_reports_each_servers_verdict(session_factory): active = _Server(broadcast="bad-txns-inputs-missingorspent") listener = _make_listener( session_factory, active, {"first.example": _Server(broadcast="ok"), "second.example": _Server(broadcast=None), "third.example": _Server(broadcast="min relay fee not met")}, ) answers = await listener.rebroadcast_everywhere("deadbeef") assert answers.active == ("refused", "bad-txns-inputs-missingorspent") assert answers.others == (("accepted", ""), None, ("refused", "min relay fee not met")) assert active.broadcasted == ["deadbeef"] async def test_confirmed_spender_needs_a_majority_of_a_quorum_of_the_other_servers(session_factory): raw, spender = _spending_tx(_TX_INPUT_TXID) mined = [{"tx_hash": spender, "height": 50}] outpoints = {(_TX_INPUT_TXID, 0)} txs = {spender: raw} agree = _make_listener( session_factory, _Server([]), # the active server's view is irrelevant {"first.example": _Server(mined, txs), "second.example": _Server(mined, txs), "third.example": _Server([])}, ) assert await agree.confirmed_spender("s", {"own"}, outpoints) == spender minority = _make_listener( session_factory, _Server(mined, txs), {"first.example": _Server(mined, txs), "second.example": _Server([]), "third.example": _Server([])}, ) assert await minority.confirmed_spender("s", {"own"}, outpoints) is None too_few = _make_listener( session_factory, _Server([]), {"first.example": _Server(mined, txs), "second.example": _Server(None), "third.example": _Server(None)}, ) assert await too_few.confirmed_spender("s", {"own"}, outpoints) is None async def test_confirmed_spender_ignores_unconfirmed_transactions_and_the_rows_own_versions(session_factory): raw, other = _spending_tx(_TX_INPUT_TXID) outpoints = {(_TX_INPUT_TXID, 0)} in_mempool = [{"tx_hash": other, "height": 0}] unconfirmed = _make_listener( session_factory, _Server([]), {"first.example": _Server(in_mempool, {other: raw}), "second.example": _Server(in_mempool, {other: raw})} ) assert await unconfirmed.confirmed_spender("s", {"own"}, outpoints) is None mined = [{"tx_hash": other, "height": 9}] own = _make_listener( session_factory, _Server([]), {"first.example": _Server(mined, {other: raw}), "second.example": _Server(mined, {other: raw})} ) assert await own.confirmed_spender("s", {other}, outpoints) is None # it is one of the row's own versions async def test_confirmed_spender_ignores_a_server_that_returns_a_different_transaction(session_factory): raw, spender = _spending_tx(_TX_INPUT_TXID) other_raw, _ = _spending_tx("33" * 32) mined = [{"tx_hash": spender, "height": 9}] lying = _make_listener( session_factory, _Server([]), {"first.example": _Server(mined, {spender: other_raw}), "second.example": _Server(mined, {spender: other_raw})}, ) assert await lying.confirmed_spender("s", set(), {(_TX_INPUT_TXID, 0)}) is None