Check confirmation/existence via scripthash history, not verbose replies (B-41)

poll_once and reconcile.py's existence check both called
blockchain.transaction.get(txid, verbose=True). Several Electrum server
implementations and versions reject the verbose flag outright
("verbose transactions are currently unsupported"), which would have
meant no confirmations and no reconciliation ever running against such
a server, read as a plain transport error. reconcile.py additionally
decided whether to abandon a transaction - releasing its funds - by
substring-matching the error text ("missing", "not found", ...), which
only works against ElectrumX's specific wording.

Both now ask blockchain.scripthash.get_history for the address that
owns every input of the transaction (a user's own address for a
bet/withdrawal, the pool address for a payout) and look for the txid in
the result: present with height > 0 means confirmed, present with
height <= 0 means still in the mempool, absent means the server
doesn't know it. get_history is a plain, universally-supported Electrum
method, and "not in the list" replaces the old substring-matching
entirely - no more guessing at error wording to decide whether to
release funds. History is cached per scripthash within one pass, since
every "payout" row shares the same pool address.

New app/tx/pending_address.py factors out own_address_for (the
address derivation was previously duplicated informally inside
tx/broadcast.py's signing context) so confirmation.py and reconcile.py
share one definition instead of two that could compute different
addresses for the same row.

tests/unit/test_confirmation.py and test_reconcile.py needed real User
rows and a master-key bootstrap they didn't have before, since address
derivation is now exercised for real rather than assumed. Suite grows
from 217 to 222 tests. BUGS.md moves B-41 to Previously fixed - no
Medium-severity finding remains open.

tests/unit/test_confirmation.py

@@ -1,41 +1,83 @@
 import pytest
-from sqlalchemy import select
 from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
 
 import app.bets.confirmation  # noqa: F401  (registers the "bet" handler)
 import app.rounds.confirmation  # noqa: F401  (registers the "payout" handler)
+from sqlalchemy import select
+
+from app.config import settings
 from app.db.base import Base
-from app.db.models import PendingTransaction, Round, RoundParticipant
+from app.db.models import PendingTransaction, Round, RoundParticipant, User
+from app.electrum.scripthash import address_to_scripthash
 from app.tx.confirmation import poll_once
+from app.wallet.hd import derive_user_address
 
 
 class FakeClient:
-    def __init__(self, confirmations_by_txid: dict[str, int]):
-        self._confirmations = confirmations_by_txid
+    """B-41: poll_once now asks blockchain.scripthash.get_history rather than a
+    verbose blockchain.transaction.get, so this hands back a flat history —
+    height > 0 means confirmed at that height, 0 (or absent) means still in the
+    mempool. The scripthash argument is ignored: every candidate's derived
+    address is looked up against the same known universe of txids, which is
+    fine since matching happens on tx_hash, not on which address asked."""
 
-    async def get_transaction(self, txid: str, verbose: bool = False) -> dict:
-        return {"confirmations": self._confirmations.get(txid, 0)}
+    def __init__(self, heights_by_txid: dict[str, int]):
+        self._heights = heights_by_txid
+
+    async def get_history(self, scripthash: str) -> list[dict]:
+        return [{"tx_hash": txid, "height": height} for txid, height in self._heights.items()]
 
 
 @pytest.fixture
-async def session_factory():
+async def session_factory(tmp_path, monkeypatch):
+    # own_address_for (B-41) derives each row's address via the HD wallet, so
+    # poll_once now needs a real master key — same bootstrap test_broadcast.py
+    # and test_reconcile.py use.
+    monkeypatch.setattr(settings, "master_key_path", str(tmp_path / "master.xprv.enc"))
+    monkeypatch.setattr(
+        settings,
+        "xprv_encryption_key",
+        __import__("cryptography.fernet", fromlist=["Fernet"]).Fernet.generate_key().decode(),
+    )
+    from app.wallet import hd
+
+    hd._account_key = None
+    hd.generate_master_key()
+
     engine = create_async_engine("sqlite+aiosqlite:///:memory:")
     async with engine.begin() as conn:
         await conn.run_sync(Base.metadata.create_all)
     yield async_sessionmaker(engine, expire_on_commit=False)
     await engine.dispose()
+    hd._account_key = None
+
+
+async def _make_user(session, derivation_index: int) -> User:
+    user = User(
+        username=f"user{derivation_index}",
+        password_hash="x",
+        derivation_index=derivation_index,
+        address=derive_user_address(derivation_index),
+    )
+    session.add(user)
+    await session.flush()
+    return user
 
 
 async def test_bet_confirmation_marks_participant_confirmed(session_factory):
     async with session_factory() as session:
+        user = await _make_user(session, 0)
         session.add(Round(id=1, status="open"))
         session.add(
             RoundParticipant(
-                round_id=1, user_id=1, bet_amount_sats=1_000, bet_txid="tx1", status="broadcast"
+                round_id=1, user_id=user.id, bet_amount_sats=1_000, bet_txid="tx1", status="broadcast"
             )
         )
         session.add(
-            PendingTransaction(kind="bet", round_id=1, user_id=1, current_txid="tx1", fee_rate_sat_vb=1, raw_tx_hex="00", status="pending")
+            PendingTransaction(
+                kind="bet", round_id=1, user_id=user.id, current_txid="tx1", fee_rate_sat_vb=1,
+                raw_tx_hex="00", status="pending",
+            )
         )
         await session.commit()
 
@@ -53,9 +95,17 @@ async def test_bet_confirmation_marks_participant_confirmed(session_factory):
 
 async def test_unconfirmed_tx_is_left_pending(session_factory):
     async with session_factory() as session:
+        user = await _make_user(session, 0)
         session.add(Round(id=2, status="open"))
-        session.add(RoundParticipant(round_id=2, user_id=1, bet_amount_sats=1_000, bet_txid="tx2", status="broadcast"))
-        session.add(PendingTransaction(kind="bet", round_id=2, user_id=1, current_txid="tx2", fee_rate_sat_vb=1, raw_tx_hex="00", status="pending"))
+        session.add(
+            RoundParticipant(round_id=2, user_id=user.id, bet_amount_sats=1_000, bet_txid="tx2", status="broadcast")
+        )
+        session.add(
+            PendingTransaction(
+                kind="bet", round_id=2, user_id=user.id, current_txid="tx2", fee_rate_sat_vb=1,
+                raw_tx_hex="00", status="pending",
+            )
+        )
         await session.commit()
 
     client = FakeClient({"tx2": 0})
@@ -70,7 +120,11 @@ async def test_unconfirmed_tx_is_left_pending(session_factory):
 async def test_payout_confirmation_closes_round(session_factory):
     async with session_factory() as session:
         session.add(Round(id=3, status="paying_out", payout_txid="tx3"))
-        session.add(PendingTransaction(kind="payout", round_id=3, current_txid="tx3", fee_rate_sat_vb=1, raw_tx_hex="00", status="pending"))
+        session.add(
+            PendingTransaction(
+                kind="payout", round_id=3, current_txid="tx3", fee_rate_sat_vb=1, raw_tx_hex="00", status="pending"
+            )
+        )
         await session.commit()
 
     client = FakeClient({"tx3": 2})
@@ -83,43 +137,52 @@ async def test_payout_confirmation_closes_round(session_factory):
 
 
 class ExplodingClient:
-    """Answers for one txid and raises for the other — a tx the server no longer
-    knows (dropped from the mempool, replaced by a bump)."""
+    """Answers for one address's history and raises for the other's — the
+    get_history equivalent of a server that no longer knows a particular tx
+    (dropped from the mempool, replaced by a bump)."""
 
-    def __init__(self, known: dict[str, int], exploding_txid: str):
-        self._known = known
-        self._exploding = exploding_txid
+    def __init__(self, heights_by_txid: dict[str, int], exploding_scripthash: str):
+        self._heights = heights_by_txid
+        self._exploding = exploding_scripthash
 
-    async def get_transaction(self, txid: str, verbose: bool = False) -> dict:
-        if txid == self._exploding:
-            raise RuntimeError("missing transaction")
-        return {"confirmations": self._known.get(txid, 0)}
+    async def get_history(self, scripthash: str) -> list[dict]:
+        if scripthash == self._exploding:
+            raise RuntimeError("server error")
+        return [{"tx_hash": txid, "height": height} for txid, height in self._heights.items()]
 
 
-async def test_one_unresolvable_txid_does_not_block_the_others(session_factory):
-    """B-03: the lookup used to be unguarded, so a single unknown txid aborted the
-    whole pass — nothing confirmed again until an operator intervened, which in turn
-    meant no round could ever close."""
+async def test_one_unresolvable_candidate_does_not_block_the_others(session_factory):
+    """B-03: the lookup used to be unguarded, so a single failing candidate aborted
+    the whole pass — nothing confirmed again until an operator intervened, which in
+    turn meant no round could ever close. B-41 changed the failure unit from "one
+    txid" to "one address's history", but the isolation guarantee is the same."""
     async with session_factory() as session:
+        good_user = await _make_user(session, 0)
+        gone_user = await _make_user(session, 1)
         session.add(Round(id=10, status="open"))
         session.add(
-            RoundParticipant(round_id=10, user_id=1, bet_amount_sats=1_000, bet_txid="good", status="broadcast")
-        )
-        session.add(
-            PendingTransaction(
-                kind="bet", round_id=10, user_id=2, current_txid="gone", fee_rate_sat_vb=1, raw_tx_hex="00",
-                status="pending",
+            RoundParticipant(
+                round_id=10, user_id=good_user.id, bet_amount_sats=1_000, bet_txid="good", status="broadcast"
             )
         )
         session.add(
             PendingTransaction(
-                kind="bet", round_id=10, user_id=1, current_txid="good", fee_rate_sat_vb=1, raw_tx_hex="00",
-                status="pending",
+                kind="bet", round_id=10, user_id=gone_user.id, current_txid="gone", fee_rate_sat_vb=1,
+                raw_tx_hex="00", status="pending",
+            )
+        )
+        session.add(
+            PendingTransaction(
+                kind="bet", round_id=10, user_id=good_user.id, current_txid="good", fee_rate_sat_vb=1,
+                raw_tx_hex="00", status="pending",
             )
         )
         await session.commit()
 
-    confirmed = await poll_once(session_factory, ExplodingClient({"good": 1}, exploding_txid="gone"))
+    exploding_scripthash = address_to_scripthash(derive_user_address(1))
+    confirmed = await poll_once(
+        session_factory, ExplodingClient({"good": 1}, exploding_scripthash=exploding_scripthash)
+    )
     assert confirmed == 1  # the healthy one still got processed
 
     async with session_factory() as session:
@@ -135,15 +198,16 @@ async def test_bet_confirms_after_an_rbf_bump_changed_the_txid(session_factory):
     a txid no participant carried — the participant stayed "broadcast" forever and
     the round could never close. It now resolves by (round_id, user_id)."""
     async with session_factory() as session:
+        user = await _make_user(session, 0)
         session.add(Round(id=11, status="open"))
         session.add(
             RoundParticipant(
-                round_id=11, user_id=7, bet_amount_sats=1_000, bet_txid="old-txid", status="broadcast"
+                round_id=11, user_id=user.id, bet_amount_sats=1_000, bet_txid="old-txid", status="broadcast"
             )
         )
         session.add(
             PendingTransaction(
-                kind="bet", round_id=11, user_id=7, current_txid="bumped-txid", fee_rate_sat_vb=2,
+                kind="bet", round_id=11, user_id=user.id, current_txid="bumped-txid", fee_rate_sat_vb=2,
                 raw_tx_hex="00", status="pending", replaced_by_txid="old-txid",
             )
         )
@@ -171,3 +235,35 @@ async def test_payout_confirms_after_an_rbf_bump_changed_the_txid(session_factor
 
     async with session_factory() as session:
         assert (await session.get(Round, 12)).status == "closed"
+
+
+async def test_poll_once_caches_history_per_scripthash(session_factory):
+    """Two pending bets from the same user share one address — fetching its
+    history twice in one pass would be wasteful."""
+    async with session_factory() as session:
+        user = await _make_user(session, 0)
+        session.add(Round(id=20, status="open"))
+        session.add(
+            PendingTransaction(
+                kind="bet", round_id=20, user_id=user.id, current_txid="tx-a", fee_rate_sat_vb=1,
+                raw_tx_hex="00", status="pending",
+            )
+        )
+        session.add(
+            PendingTransaction(
+                kind="withdrawal", user_id=user.id, current_txid="tx-b", fee_rate_sat_vb=1,
+                raw_tx_hex="00", status="pending",
+            )
+        )
+        await session.commit()
+
+    call_count = {"n": 0}
+
+    class CountingClient:
+        async def get_history(self, scripthash: str) -> list[dict]:
+            call_count["n"] += 1
+            return [{"tx_hash": "tx-a", "height": 0}, {"tx_hash": "tx-b", "height": 0}]
+
+    await poll_once(session_factory, CountingClient())
+
+    assert call_count["n"] == 1

tests/unit/test_pending_address.py

@@ -0,0 +1,55 @@
+"""B-41: own_address_for is the single place tx/confirmation.py and
+tx/reconcile.py derive a PendingTransaction's own address from — a payout's
+address must always be the pool's, everything else the actual user's."""
+
+import pytest
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.config import settings
+from app.db.base import Base
+from app.db.models import User
+from app.tx.pending_address import own_address_for
+
+
+@pytest.fixture
+async def session_factory(tmp_path, monkeypatch):
+    monkeypatch.setattr(settings, "master_key_path", str(tmp_path / "master.xprv.enc"))
+    monkeypatch.setattr(
+        settings,
+        "xprv_encryption_key",
+        __import__("cryptography.fernet", fromlist=["Fernet"]).Fernet.generate_key().decode(),
+    )
+    from app.wallet import hd
+
+    hd._account_key = None
+    hd.generate_master_key()
+
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+    async with engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    yield async_sessionmaker(engine, expire_on_commit=False)
+    await engine.dispose()
+    hd._account_key = None
+
+
+async def test_payout_uses_the_pool_address_regardless_of_user_id(session_factory):
+    from app.wallet.hd import derive_pool_address
+
+    async with session_factory() as session:
+        address = await own_address_for(session, "payout", None)
+
+    assert address == derive_pool_address()
+
+
+@pytest.mark.parametrize("kind", ["bet", "withdrawal"])
+async def test_bet_and_withdrawal_use_the_users_own_address(session_factory, kind):
+    from app.wallet.hd import derive_user_address
+
+    async with session_factory() as session:
+        user = User(username="alice", password_hash="x", derivation_index=3, address=derive_user_address(3))
+        session.add(user)
+        await session.flush()
+
+        address = await own_address_for(session, kind, user.id)
+
+    assert address == derive_user_address(3)

tests/unit/test_reconcile.py

@@ -1,5 +1,10 @@
 """Regression tests for B-04 (and the "building" half of B-08): a transaction that
-never made it onto the chain must give the coins back instead of freezing them."""
+never made it onto the chain must give the coins back instead of freezing them.
+
+Also covers B-41: existence/reconciliation checks go through
+blockchain.scripthash.get_history rather than a verbose blockchain.transaction.get
+reply, so the fake clients below implement get_history.
+"""
 
 import pytest
 from embit import script
@@ -7,37 +12,60 @@ from embit.transaction import Transaction, TransactionInput, TransactionOutput
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
 
+from app.config import settings
 from app.db.base import Base
 from app.db.models import AuditLog, PendingTransaction, RoundParticipant, User, UtxoEvent, Withdrawal
 from app.tx.reconcile import reconcile_once
 
 
 class UnknownTxClient:
-    """A server that doesn't know any of the txids it's asked about."""
+    """A server whose history for any address never includes our txid."""
 
-    async def get_transaction(self, txid: str, verbose: bool = False):
-        raise RuntimeError(f"missing transaction {txid}")
+    async def get_history(self, scripthash: str) -> list[dict]:
+        return []
 
 
 class KnownTxClient:
-    async def get_transaction(self, txid: str, verbose: bool = False):
-        return {"txid": txid, "confirmations": 0}
+    """A server whose history for the address includes our txid — mined or
+    still in the mempool doesn't matter for existence, only for confirmation
+    (which is tx/confirmation.py's concern, not reconcile.py's)."""
+
+    def __init__(self, txid: str = "betxid"):
+        self._txid = txid
+
+    async def get_history(self, scripthash: str) -> list[dict]:
+        return [{"tx_hash": self._txid, "height": 100}]
 
 
 class BrokenClient:
     """A transport failure — says nothing about whether the tx exists."""
 
-    async def get_transaction(self, txid: str, verbose: bool = False):
+    async def get_history(self, scripthash: str) -> list[dict]:
         raise ConnectionResetError("connection reset")
 
 
 @pytest.fixture
-async def session_factory():
+async def session_factory(tmp_path, monkeypatch):
+    # own_address_for (B-41) derives each row's address via the HD wallet rather
+    # than trusting the DB's address column, so reconcile_once now needs a real
+    # master key set up — same bootstrap test_broadcast.py uses.
+    monkeypatch.setattr(settings, "master_key_path", str(tmp_path / "master.xprv.enc"))
+    monkeypatch.setattr(
+        settings,
+        "xprv_encryption_key",
+        __import__("cryptography.fernet", fromlist=["Fernet"]).Fernet.generate_key().decode(),
+    )
+    from app.wallet import hd
+
+    hd._account_key = None
+    hd.generate_master_key()
+
     engine = create_async_engine("sqlite+aiosqlite:///:memory:")
     async with engine.begin() as conn:
         await conn.run_sync(Base.metadata.create_all)
     yield async_sessionmaker(engine, expire_on_commit=False)
     await engine.dispose()
+    hd._account_key = None
 
 
 # A real (unsigned) transaction spending one input, built rather than hand-written
@@ -66,11 +94,19 @@ async def _seed_bet(
     participant_status: str,
     age_seconds: int,
     last_broadcast_age_seconds: int | None = None,
+    derivation_index: int = 0,
 ):
     from datetime import datetime, timedelta, timezone
 
+    from app.wallet.hd import derive_user_address
+
     async with session_factory() as session:
-        user = User(username="u", password_hash="x", derivation_index=0, address="plm1qtest")
+        user = User(
+            username="u",
+            password_hash="x",
+            derivation_index=derivation_index,
+            address=derive_user_address(derivation_index),
+        )
         session.add(user)
         await session.flush()
         session.add(
@@ -145,7 +181,7 @@ async def test_promotes_a_building_row_whose_tx_did_reach_the_chain(session_fact
         session_factory, pending_status="building", participant_status="building", age_seconds=300
     )
 
-    resolved = await reconcile_once(session_factory, KnownTxClient())
+    resolved = await reconcile_once(session_factory, KnownTxClient("betxid"))
     assert resolved == 1
 
     async with session_factory() as session:
@@ -219,8 +255,10 @@ async def test_abandoned_withdrawal_is_marked_failed_and_kept(session_factory):
     they can see it didn't go through."""
     from datetime import datetime, timedelta, timezone
 
+    from app.wallet.hd import derive_user_address
+
     async with session_factory() as session:
-        user = User(username="w", password_hash="x", derivation_index=1, address="plm1qtest2")
+        user = User(username="w", password_hash="x", derivation_index=1, address=derive_user_address(1))
         session.add(user)
         await session.flush()
         session.add(
@@ -264,3 +302,42 @@ async def test_abandoned_withdrawal_is_marked_failed_and_kept(session_factory):
         assert withdrawal.status == "failed"
         assert withdrawal.txid is None
         assert (await session.scalars(select(UtxoEvent))).one().spent_txid is None
+
+
+# --- B-41: existence checks now use get_history and share it across candidates
+# sharing the same address, instead of a per-tx verbose blockchain.transaction.get. --
+
+
+async def test_reconcile_once_caches_history_per_scripthash(session_factory):
+    """Two payout PendingTransaction rows always share the same pool address —
+    fetching its history twice in one pass would be wasteful and, at scale
+    across many candidates on one address, needlessly slow the whole tick."""
+    from datetime import datetime, timedelta, timezone
+
+    async with session_factory() as session:
+        old = datetime.now(timezone.utc) - timedelta(hours=7)
+        session.add(
+            PendingTransaction(
+                kind="payout", round_id=1, current_txid="payout-a", fee_rate_sat_vb=1,
+                raw_tx_hex=_RAW_TX, status="pending", broadcast_at=old, last_broadcast_at=old,
+            )
+        )
+        session.add(
+            PendingTransaction(
+                kind="payout", round_id=2, current_txid="payout-b", fee_rate_sat_vb=1,
+                raw_tx_hex=_RAW_TX, status="pending", broadcast_at=old, last_broadcast_at=old,
+            )
+        )
+        await session.commit()
+
+    call_count = {"n": 0}
+
+    class CountingClient:
+        async def get_history(self, scripthash: str) -> list[dict]:
+            call_count["n"] += 1
+            return [{"tx_hash": "payout-a", "height": 100}, {"tx_hash": "payout-b", "height": 100}]
+
+    resolved = await reconcile_once(session_factory, CountingClient())
+
+    assert resolved == 0  # both exist — nothing to abandon or promote (already "pending")
+    assert call_count["n"] == 1  # one call covered both rows sharing the pool address