bkpr: helpers to query sql plugin for chainmoves and channelmoves.

We're going to be using this instead of our internal db.

I also made json_out_obj() take the str arg, as it didn't and I
expected it to.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
Rusty Russell
2025-08-19 10:30:51 +09:30
parent 2372dbda30
commit 45e860ad58
6 changed files with 476 additions and 3 deletions

View File

@@ -13,7 +13,8 @@ BOOKKEEPER_PLUGIN_SRC := \
plugins/bkpr/incomestmt.c \
plugins/bkpr/onchain_fee.c \
plugins/bkpr/rebalances.c \
plugins/bkpr/recorder.c
plugins/bkpr/recorder.c \
plugins/bkpr/sql.c
BOOKKEEPER_DB_QUERIES := \
plugins/bkpr/db_sqlite3_sqlgen.c \
@@ -33,7 +34,8 @@ BOOKKEEPER_HEADER := \
plugins/bkpr/incomestmt.h \
plugins/bkpr/onchain_fee.h \
plugins/bkpr/rebalances.h \
plugins/bkpr/recorder.h
plugins/bkpr/recorder.h \
plugins/bkpr/sql.h
BOOKKEEPER_OBJS := $(BOOKKEEPER_SRC:.c=.o)

218
plugins/bkpr/sql.c Normal file
View File

@@ -0,0 +1,218 @@
#include "config.h"
#include <common/json_stream.h>
#include <plugins/bkpr/blockheights.h>
#include <plugins/bkpr/chain_event.h>
#include <plugins/bkpr/channel_event.h>
#include <plugins/bkpr/sql.h>
#include <plugins/libplugin.h>
const jsmntok_t *sql_req(const tal_t *ctx,
struct command *cmd,
const char **buf,
const char *fmt, ...)
{
va_list ap;
const jsmntok_t *ret;
va_start(ap, fmt);
ret = sql_reqv(ctx, cmd, buf, fmt, ap);
va_end(ap);
return ret;
}
const jsmntok_t *sql_reqv(const tal_t *ctx,
struct command *cmd,
const char **buf,
const char *fmt, va_list ap)
{
struct json_out *params;
params = json_out_obj(NULL, "query", take(tal_vfmt(NULL, fmt, ap)));
return jsonrpc_request_sync(ctx, cmd, "sql", take(params), buf);
}
static struct channel_event **channel_events(const tal_t *ctx,
const char *buf,
const jsmntok_t *result)
{
struct channel_event **evs;
size_t i;
const jsmntok_t *row, *rows = json_get_member(buf, result, "rows");
evs = tal_arr(ctx, struct channel_event *, rows->size);
json_for_each_arr(i, row, rows) {
bool ok = true;
struct channel_event *ev;
u64 created_index;
const char *account_name;
const char *primary_tag;
struct amount_msat credit, debit, fees;
struct sha256 *payment_id, payment_hash;
u64 part_id, timestamp;
const jsmntok_t *val = row + 1;
assert(row->size == 9);
ok &= json_to_u64(buf, val, &created_index);
val = json_next(val);
account_name = json_strdup(NULL, buf, val);
val = json_next(val);
primary_tag = json_strdup(NULL, buf, val);
val = json_next(val);
ok &= json_to_msat(buf, val, &credit);
val = json_next(val);
ok &= json_to_msat(buf, val, &debit);
val = json_next(val);
ok &= json_to_msat(buf, val, &fees);
val = json_next(val);
if (json_tok_is_null(buf, val))
payment_id = NULL;
else {
ok &= json_to_sha256(buf, val, &payment_hash);
payment_id = &payment_hash;
}
val = json_next(val);
if (json_tok_is_null(buf, val))
part_id = 0;
else {
ok &= json_to_u64(buf, val, &part_id);
}
val = json_next(val);
ok &= json_to_u64(buf, val, &timestamp);
assert(ok);
ev = new_channel_event(evs,
take(primary_tag),
credit, debit, fees,
payment_id,
part_id,
timestamp);
ev->acct_name = tal_steal(ev, account_name);
ev->db_id = created_index;
evs[i] = ev;
}
return evs;
}
static struct chain_event **chain_events(const tal_t *ctx,
const struct bkpr *bkpr,
const char *buf,
const jsmntok_t *result)
{
struct chain_event **evs;
size_t i;
const jsmntok_t *row, *rows = json_get_member(buf, result, "rows");
evs = tal_arr(ctx, struct chain_event *, rows->size);
json_for_each_arr(i, row, rows) {
bool ok = true;
struct chain_event *ev = tal(evs, struct chain_event);
int flag;
const jsmntok_t *val = row + 1;
assert(row->size == 14);
ok &= json_to_u64(buf, val, &ev->db_id);
val = json_next(val);
ev->acct_name = json_strdup(ev, buf, val);
val = json_next(val);
if (json_tok_is_null(buf, val))
ev->origin_acct = NULL;
else
ev->origin_acct = json_strdup(ev, buf, val);
val = json_next(val);
ev->tag = json_strdup(ev, buf, val);
val = json_next(val);
ok &= json_to_msat(buf, val, &ev->credit);
val = json_next(val);
ok &= json_to_msat(buf, val, &ev->debit);
val = json_next(val);
ok &= json_to_msat(buf, val, &ev->output_value);
val = json_next(val);
ok &= json_to_u64(buf, val, &ev->timestamp);
val = json_next(val);
ok &= json_to_u32(buf, val, &ev->blockheight);
val = json_next(val);
ok &= json_to_outpoint(buf, val, &ev->outpoint);
/* We may know better! */
if (ev->blockheight == 0)
ev->blockheight = find_blockheight(bkpr, &ev->outpoint.txid);
val = json_next(val);
if (json_tok_is_null(buf, val))
ev->spending_txid = NULL;
else {
ev->spending_txid = tal(ev, struct bitcoin_txid);
ok &= json_to_txid(buf, val, ev->spending_txid);
}
val = json_next(val);
if (json_tok_is_null(buf, val))
ev->payment_id = NULL;
else {
ev->payment_id = tal(ev, struct sha256);
ok &= json_to_sha256(buf, val, ev->payment_id);
}
val = json_next(val);
/* These are 0/1 not true/false */
ok &= json_to_int(buf, val, &flag);
ev->stealable = flag;
val = json_next(val);
/* These are 0/1 not true/false */
ok &= json_to_int(buf, val, &flag);
ev->splice_close = flag;
assert(ok);
evs[i] = ev;
}
return evs;
}
struct channel_event **
channel_events_from_sql(const tal_t *ctx,
struct command *cmd,
const char *fmt, ...)
{
va_list ap;
const jsmntok_t *toks;
const char *buf;
va_start(ap, fmt);
toks = sql_reqv(tmpctx, cmd, &buf, fmt, ap);
va_end(ap);
return channel_events(ctx, buf, toks);
}
struct chain_event **
chain_events_from_sql(const tal_t *ctx,
const struct bkpr *bkpr,
struct command *cmd,
const char *fmt, ...)
{
va_list ap;
const jsmntok_t *toks;
const char *buf;
va_start(ap, fmt);
toks = sql_reqv(tmpctx, cmd, &buf, fmt, ap);
va_end(ap);
return chain_events(ctx, bkpr, buf, toks);
}
const char *sql_string(const tal_t *ctx, const char *str)
{
char *ret;
size_t out;
if (!strchr(str, '\''))
return str;
/* Worst case size */
ret = tal_arr(ctx, char, strlen(str) * 2 + 1);
out = 0;
for (size_t in = 0; str[in]; in++) {
ret[out++] = str[in];
if (str[in] == '\'')
ret[out++] = str[in];
}
ret[out] = '\0';
return ret;
}

70
plugins/bkpr/sql.h Normal file
View File

@@ -0,0 +1,70 @@
#ifndef LIGHTNING_PLUGINS_BKPR_SQL_H
#define LIGHTNING_PLUGINS_BKPR_SQL_H
#include "config.h"
#include <ccan/tal/str/str.h>
#include <common/json_parse_simple.h>
struct command;
struct command_result;
#define SELECT_CHANNEL_EVENTS \
"SELECT" \
" created_index" \
", account_id" \
", primary_tag" \
", credit_msat" \
", debit_msat" \
", fees_msat" \
", payment_hash" \
", part_id" \
", timestamp" \
" FROM channelmoves "
#define SELECT_CHAIN_EVENTS \
"SELECT" \
" created_index" \
", account_id" \
", originating_account" \
", primary_tag" \
", credit_msat" \
", debit_msat" \
", output_msat" \
", timestamp" \
", blockheight" \
", utxo" \
", spending_txid" \
", payment_hash" \
", EXISTS (SELECT 1 FROM chainmoves_extra_tags et" \
" WHERE et.row = cm.rowid" \
" AND et.extra_tags = 'stealable') AS stealable" \
", EXISTS (SELECT 1 FROM chainmoves_extra_tags et" \
" WHERE et.row = cm.rowid" \
" AND et.extra_tags = 'splice') AS spliced" \
" FROM chainmoves cm "
const jsmntok_t *
PRINTF_FMT(4, 5) sql_req(const tal_t *ctx,
struct command *cmd, const char **buf,
const char *fmt, ...);
const jsmntok_t *sql_reqv(const tal_t *ctx,
struct command *cmd, const char **buf,
const char *fmt, va_list ap);
struct channel_event **
PRINTF_FMT(3, 4) channel_events_from_sql(const tal_t *ctx,
struct command *cmd,
const char *fmt, ...);
struct chain_event **
PRINTF_FMT(4, 5) chain_events_from_sql(const tal_t *ctx,
const struct bkpr *bkpr,
struct command *cmd,
const char *fmt, ...);
/* FIXME: The sql plugin should support bound parameters to avoid this! */
/* Return with escaped quotes, if any */
const char *sql_string(const tal_t *ctx, const char *str);
#endif /* LIGHTNING_PLUGINS_BKPR_SQL_H */

181
plugins/bkpr/test/run-sql.c Normal file
View File

@@ -0,0 +1,181 @@
#include "config.h"
#include "plugins/bkpr/sql.c"
#include "plugins/libplugin.c"
#include <common/json_filter.h>
#include <common/json_stream.h>
#include <common/fee_states.h>
#include <common/setup.h>
#include <common/utils.h>
#include <stdio.h>
#include <wire/wire.h>
/* AUTOGENERATED MOCKS START */
/* Generated stub for daemon_developer_mode */
bool daemon_developer_mode(char *argv[])
{ fprintf(stderr, "daemon_developer_mode called!\n"); abort(); }
/* Generated stub for daemon_setup */
void daemon_setup(const char *argv0 UNNEEDED,
void (*backtrace_print)(const char *fmt UNNEEDED, ...) UNNEEDED,
void (*backtrace_exit)(void))
{ fprintf(stderr, "daemon_setup called!\n"); abort(); }
/* Generated stub for deprecated_ok_ */
bool deprecated_ok_(bool deprecated_apis UNNEEDED,
const char *feature UNNEEDED,
const char *start UNNEEDED,
const char *end UNNEEDED,
const char **begs UNNEEDED,
void (*complain)(const char *feat UNNEEDED, bool allowing UNNEEDED, void *) UNNEEDED,
void *cbarg UNNEEDED)
{ fprintf(stderr, "deprecated_ok_ called!\n"); abort(); }
/* Generated stub for find_blockheight */
u32 find_blockheight(const struct bkpr *bkpr UNNEEDED, const struct bitcoin_txid *txid UNNEEDED)
{ fprintf(stderr, "find_blockheight called!\n"); abort(); }
/* Generated stub for first_fee_state */
enum htlc_state first_fee_state(enum side opener UNNEEDED)
{ fprintf(stderr, "first_fee_state called!\n"); abort(); }
/* Generated stub for fmt_wireaddr_without_port */
char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED)
{ fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); }
/* Generated stub for fromwire_wireaddr */
bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED)
{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); }
/* Generated stub for htlc_state_flags */
int htlc_state_flags(enum htlc_state state UNNEEDED)
{ fprintf(stderr, "htlc_state_flags called!\n"); abort(); }
/* Generated stub for htlc_state_name */
const char *htlc_state_name(enum htlc_state s UNNEEDED)
{ fprintf(stderr, "htlc_state_name called!\n"); abort(); }
/* Generated stub for is_asterix_notification */
bool is_asterix_notification(const char *notification_name UNNEEDED,
const char *subscriptions UNNEEDED)
{ fprintf(stderr, "is_asterix_notification called!\n"); abort(); }
/* Generated stub for json_filter_down */
bool json_filter_down(struct json_filter **filter UNNEEDED, const char *member UNNEEDED)
{ fprintf(stderr, "json_filter_down called!\n"); abort(); }
/* Generated stub for json_filter_finished */
bool json_filter_finished(const struct json_filter *filter UNNEEDED)
{ fprintf(stderr, "json_filter_finished called!\n"); abort(); }
/* Generated stub for json_filter_misused */
const char *json_filter_misused(const tal_t *ctx UNNEEDED, const struct json_filter *f UNNEEDED)
{ fprintf(stderr, "json_filter_misused called!\n"); abort(); }
/* Generated stub for json_filter_ok */
bool json_filter_ok(const struct json_filter *filter UNNEEDED, const char *member UNNEEDED)
{ fprintf(stderr, "json_filter_ok called!\n"); abort(); }
/* Generated stub for json_filter_up */
bool json_filter_up(struct json_filter **filter UNNEEDED)
{ fprintf(stderr, "json_filter_up called!\n"); abort(); }
/* Generated stub for json_scan */
const char *json_scan(const tal_t *ctx UNNEEDED,
const char *buffer UNNEEDED,
const jsmntok_t *tok UNNEEDED,
const char *guide UNNEEDED,
...)
{ fprintf(stderr, "json_scan called!\n"); abort(); }
/* Generated stub for json_scanv */
const char *json_scanv(const tal_t *ctx UNNEEDED,
const char *buffer UNNEEDED,
const jsmntok_t *tok UNNEEDED,
const char *guide UNNEEDED,
va_list ap UNNEEDED)
{ fprintf(stderr, "json_scanv called!\n"); abort(); }
/* Generated stub for json_to_int */
bool json_to_int(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, int *num UNNEEDED)
{ fprintf(stderr, "json_to_int called!\n"); abort(); }
/* Generated stub for json_to_msat */
bool json_to_msat(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
struct amount_msat *msat UNNEEDED)
{ fprintf(stderr, "json_to_msat called!\n"); abort(); }
/* Generated stub for json_to_node_id */
bool json_to_node_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
struct node_id *id UNNEEDED)
{ fprintf(stderr, "json_to_node_id called!\n"); abort(); }
/* Generated stub for json_to_number */
bool json_to_number(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
unsigned int *num UNNEEDED)
{ fprintf(stderr, "json_to_number called!\n"); abort(); }
/* Generated stub for json_to_outpoint */
bool json_to_outpoint(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
struct bitcoin_outpoint *op UNNEEDED)
{ fprintf(stderr, "json_to_outpoint called!\n"); abort(); }
/* Generated stub for json_to_secret */
bool json_to_secret(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, struct secret *dest UNNEEDED)
{ fprintf(stderr, "json_to_secret called!\n"); abort(); }
/* Generated stub for json_to_sha256 */
bool json_to_sha256(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, struct sha256 *dest UNNEEDED)
{ fprintf(stderr, "json_to_sha256 called!\n"); abort(); }
/* Generated stub for json_to_short_channel_id */
bool json_to_short_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
struct short_channel_id *scid UNNEEDED)
{ fprintf(stderr, "json_to_short_channel_id called!\n"); abort(); }
/* Generated stub for json_to_txid */
bool json_to_txid(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
struct bitcoin_txid *txid UNNEEDED)
{ fprintf(stderr, "json_to_txid called!\n"); abort(); }
/* Generated stub for json_to_u16 */
bool json_to_u16(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
uint16_t *num UNNEEDED)
{ fprintf(stderr, "json_to_u16 called!\n"); abort(); }
/* Generated stub for json_tok_bin_from_hex */
u8 *json_tok_bin_from_hex(const tal_t *ctx UNNEEDED, const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED)
{ fprintf(stderr, "json_tok_bin_from_hex called!\n"); abort(); }
/* Generated stub for last_fee_state */
enum htlc_state last_fee_state(enum side opener UNNEEDED)
{ fprintf(stderr, "last_fee_state called!\n"); abort(); }
/* Generated stub for log_level_name */
const char *log_level_name(enum log_level level UNNEEDED)
{ fprintf(stderr, "log_level_name called!\n"); abort(); }
/* Generated stub for new_channel_event */
struct channel_event *new_channel_event(const tal_t *ctx UNNEEDED,
const char *tag UNNEEDED,
struct amount_msat credit UNNEEDED,
struct amount_msat debit UNNEEDED,
struct amount_msat fees UNNEEDED,
struct sha256 *payment_id STEALS UNNEEDED,
u32 part_id UNNEEDED,
u64 timestamp UNNEEDED)
{ fprintf(stderr, "new_channel_event called!\n"); abort(); }
/* Generated stub for param_check */
bool param_check(struct command *cmd UNNEEDED,
const char *buffer UNNEEDED,
const jsmntok_t tokens[] UNNEEDED, ...)
{ fprintf(stderr, "param_check called!\n"); abort(); }
/* Generated stub for parse_filter */
struct command_result *parse_filter(struct command *cmd UNNEEDED,
const char *name UNNEEDED,
const char *buffer UNNEEDED,
const jsmntok_t *tok UNNEEDED)
{ fprintf(stderr, "parse_filter called!\n"); abort(); }
/* AUTOGENERATED MOCKS END */
int main(int argc, char *argv[])
{
common_setup(argv[0]);
/* No quote: should return same pointer */
const char *s1 = "simple";
const char *r1 = sql_string(tmpctx, s1);
assert(r1 == s1);
/* One quote: should return new string with doubled quote */
const char *s2 = "O'Reilly";
const char *r2 = sql_string(tmpctx, s2);
assert(strcmp(r2, "O''Reilly") == 0);
assert(r2 != s2); // New allocation
/* Multiple quotes */
const char *s3 = "'a'b'c'";
const char *r3 = sql_string(tmpctx, s3);
assert(strcmp(r3, "''a''b''c''") == 0);
/* All quotes */
const char *s4 = "''''";
const char *r4 = sql_string(tmpctx, s4);
assert(strcmp(r4, "''''''''") == 0);
/* Empty string: should return same pointer */
const char *s5 = "";
const char *r5 = sql_string(tmpctx, s5);
assert(r5 == s5);
common_shutdown();
}

View File

@@ -553,6 +553,8 @@ struct json_out *json_out_obj(const tal_t *ctx,
json_out_start(jout, NULL, '{');
if (str)
json_out_addstr(jout, fieldname, str);
if (taken(str))
tal_free(str);
json_out_end(jout, '}');
json_out_finished(jout);

View File

@@ -369,7 +369,7 @@ struct command_result *command_still_pending(struct command *cmd)
* object is empty. */
struct json_out *json_out_obj(const tal_t *ctx,
const char *fieldname,
const char *str);
const char *str TAKES);
/* Return this iff the param() call failed in your handler. */
struct command_result *command_param_failed(void);