From 873d4102c8f903e795b20ecbb1f751c6e2eeb2dd Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Thu, 20 Nov 2025 12:07:15 +1030
Subject: [PATCH] bookkeeper: restore limit on asking for all channelmoves at
 once.

Now we've found all the issues, the latency spike (4 seconds on my laptop)
for querying 2M elements remains.

Restore the limited sampling which we reverted, but make it 10,000 now.

This doesn't help our worst-case latency, because sql still asks for all 2M entries on
first access.  We address that next.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 plugins/bkpr/bookkeeper.c | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/plugins/bkpr/bookkeeper.c b/plugins/bkpr/bookkeeper.c
index 579483111..87caf8786 100644
--- a/plugins/bkpr/bookkeeper.c
+++ b/plugins/bkpr/bookkeeper.c
@@ -55,6 +55,10 @@ static struct refresh_info *use_rinfo(struct refresh_info *rinfo)
 	return rinfo;
 }
 
+/* Recursion */
+static struct command_result *limited_listchannelmoves(struct command *cmd,
+						       struct refresh_info *rinfo);
+
 static struct command_result *rinfo_one_done(struct command *cmd,
 					     struct refresh_info *rinfo)
 {
@@ -105,6 +109,7 @@ static struct fee_sum *find_sum_for_txid(struct fee_sum **sums,
 	return NULL;
 }
 
+#define LISTCHANNELMOVES_LIMIT 10000
 static struct command_result *listchannelmoves_done(struct command *cmd,
 						    const char *method,
 						    const char *buf,
@@ -126,16 +131,36 @@ static struct command_result *listchannelmoves_done(struct command *cmd,
 				     "create-or-replace",
 				     datastore_done, NULL, use_rinfo(rinfo));
 
+	/* If there might be more, try asking for more */
+	if (moves->size == LISTCHANNELMOVES_LIMIT)
+		limited_listchannelmoves(cmd, rinfo);
+
 	return rinfo_one_done(cmd, rinfo);
 }
 
+/* We do 1000 at a time to avoid overwhelming lightningd */
+static struct command_result *limited_listchannelmoves(struct command *cmd,
+						       struct refresh_info *rinfo)
+{
+	struct bkpr *bkpr = bkpr_of(cmd->plugin);
+	struct out_req *req;
+
+	req = jsonrpc_request_start(cmd, "listchannelmoves",
+				    listchannelmoves_done,
+				    plugin_broken_cb,
+				    use_rinfo(rinfo));
+	json_add_string(req->js, "index", "created");
+	json_add_u64(req->js, "start", bkpr->channelmoves_index + 1);
+	json_add_u64(req->js, "limit", LISTCHANNELMOVES_LIMIT);
+	return send_outreq(req);
+}
+
 static struct command_result *listchainmoves_done(struct command *cmd,
 						  const char *method,
 						  const char *buf,
 						  const jsmntok_t *result,
 						  struct refresh_info *rinfo)
 {
-	struct out_req *req;
 	const jsmntok_t *moves, *t;
 	size_t i;
 	struct bkpr *bkpr = bkpr_of(cmd->plugin);
@@ -151,13 +176,7 @@ static struct command_result *listchainmoves_done(struct command *cmd,
 				     "create-or-replace",
 				     datastore_done, NULL, use_rinfo(rinfo));
 
-	req = jsonrpc_request_start(cmd, "listchannelmoves",
-				    listchannelmoves_done,
-				    plugin_broken_cb,
-				    use_rinfo(rinfo));
-	json_add_string(req->js, "index", "created");
-	json_add_u64(req->js, "start", bkpr->channelmoves_index + 1);
-	send_outreq(req);
+	limited_listchannelmoves(cmd, rinfo);
 	return rinfo_one_done(cmd, rinfo);
 }