davide/palladum-lightning
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

lightningd: handle large numbers of command outputs gracefully.

Browse Source 
Profiling shows us spending all our time in tal_arr_remove when dealing
with a giant number of output streams.  This applies both for RPC output
and plugin output.

Use linked list instead.

tests/test_coinmoves.py::test_generate_coinmoves (2,000,000, sqlite3):
	Time (from start to end of l2 node):	239 seconds **WAS 518**
	Worst latency:				56.9 seconds **WAS 353**

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
Rusty Russell
2025-11-20 12:07:12 +10:30
parent 6006467824
commit 8707b7312a
4 changed files with 27 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
lightningd/plugin.c
View File

@@ -372,7 +372,7 @@ struct plugin *plugin_register(struct plugins *plugins, const char* path TAKES,
p->can_check = false;
p->plugin_state = UNCONFIGURED;
p->js_arr = tal_arr(p, struct json_stream *, 0);
list_head_init(&p->jsouts);
p->notification_topics = tal_arr(p, const char *, 0);
p->subscriptions = NULL;
p->dynamic = false;
@@ -474,8 +474,8 @@ void plugin_kill(struct plugin *plugin, enum log_level loglevel,
*/
static void plugin_send(struct plugin *plugin, struct json_stream *stream)
{
tal_steal(plugin->js_arr, stream);
tal_arr_expand(&plugin->js_arr, stream);
tal_steal(plugin, stream);
list_add_tail(&plugin->jsouts, &stream->list);
io_wake(plugin);
}
@@ -816,10 +816,7 @@ static struct io_plan *plugin_write_json(struct io_conn *conn,
static struct io_plan *plugin_stream_complete(struct io_conn *conn, struct json_stream *js, struct plugin *plugin)
{
assert(tal_count(plugin->js_arr) > 0);
/* Remove js and shift all remainig over */
tal_arr_remove(&plugin->js_arr, 0);
list_del_from(&plugin->jsouts, &js->list);
/* It got dropped off the queue, free it. */
tal_free(js);
@@ -829,8 +826,11 @@ static struct io_plan *plugin_stream_complete(struct io_conn *conn, struct json_
static struct io_plan *plugin_write_json(struct io_conn *conn,
struct plugin *plugin)
{
if (tal_count(plugin->js_arr)) {
return json_stream_output(plugin->js_arr[0], plugin->stdin_conn, plugin_stream_complete, plugin);
struct json_stream *js;
js = list_top(&plugin->jsouts, struct json_stream, list);
if (js) {
return json_stream_output(js, plugin->stdin_conn, plugin_stream_complete, plugin);
}
return io_out_wait(conn, plugin, plugin_write_json, plugin);