From 99e8e9246e57f8c895ff651c44b95fc8c7dec4e0 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 7 Aug 2024 10:15:55 +0930 Subject: [PATCH] devtools/gossmap-compress: use transparent zlib compression if available. Before: ``` -rw-rw-r-- 1 rusty rusty 1643258 Jul 26 09:51 compressed ``` After: ``` -rw-rw-r-- 1 rusty rusty 508332 Jul 26 09:49 compressed ``` Signed-off-by: Rusty Russell --- Makefile | 2 +- configure | 14 ++++ devtools/Makefile | 5 ++ devtools/gossmap-compress.c | 123 ++++++++++++++++++++++-------------- 4 files changed, 94 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index adbd60098..bbe22dad9 100644 --- a/Makefile +++ b/Makefile @@ -658,7 +658,7 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o # uses some ccan modules internally). We want to rely on -lwallycore etc. # (as per EXTERNAL_LDLIBS) so we filter them out here. $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS): - @$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a -o $@) + @$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@) # We special case the fuzzing target binaries, as they need to link against libfuzzer, # which brings its own main(). diff --git a/configure b/configure index cba529fbd..3819dc1e1 100755 --- a/configure +++ b/configure @@ -367,6 +367,20 @@ trap "rm -f $CONFIG_VAR_FILE.$$" 0 $CONFIGURATOR --extra-tests --autotools-style --var-file=$CONFIG_VAR_FILE.$$ --header-file=$CONFIG_HEADER.$$ --configurator-cc="$CONFIGURATOR_CC" --wrapper="$CONFIGURATOR_WRAPPER" "$CC" ${CWARNFLAGS-$BASE_WARNFLAGS} $CDEBUGFLAGS $COPTFLAGS $CSANFLAGS -I$CPATH -L$LIBRARY_PATH $SQLITE3_CFLAGS $POSTGRES_INCLUDE < +#include + +int main(void) +{ + gzFile f = gzopen("/dev/null", "wb"); + return f != NULL ? 0 : 1; +} +/*END*/ var=HAVE_GOOD_LIBSODIUM desc=libsodium with IETF chacha20 variants style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE diff --git a/devtools/Makefile b/devtools/Makefile index f1caaa2a2..d7f86f9d7 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -11,6 +11,11 @@ ALL_C_SOURCES += $(DEVTOOLS_TOOL_SRC) ALL_C_HEADERS += ALL_PROGRAMS += $(DEVTOOLS) +# gossmap-compress wants -lz if we say we have it. +ifeq ($(HAVE_ZLIB),1) +devtools/gossmap-compress_LDLIBS=-lz +endif # ZLIB + DEVTOOLS_COMMON_OBJS := \ common/amount.o \ common/autodata.o \ diff --git a/devtools/gossmap-compress.c b/devtools/gossmap-compress.c index 2683aa98d..5da423bf3 100644 --- a/devtools/gossmap-compress.c +++ b/devtools/gossmap-compress.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -16,8 +15,29 @@ #include #include #include +#include #include #include +#if HAVE_ZLIB +#include +#else +/* Worst... zlib... ever! */ +#define gzFile int +#define gzdopen(fd, mode) (fd) +#define gzclose(outf) close(outf) +static int gzread(int fd, void *buf, size_t len) +{ + if (read_all(fd, buf, len)) + return len; + return 0; +} +static int gzwrite(int fd, const void *buf, size_t len) +{ + if (write_all(fd, buf, len)) + return len; + return 0; +} +#endif static unsigned int verbose = 0; @@ -70,35 +90,35 @@ static int cmp_node_num_chans(struct gossmap_node *const *a, return (int)(*a)->num_chans - (int)(*b)->num_chans; } -static void write_bigsize(int outfd, u64 val) +static void write_bigsize(gzFile outf, u64 val) { u8 buf[BIGSIZE_MAX_LEN]; size_t len; len = bigsize_put(buf, val); - if (!write_all(outfd, buf, len)) - errx(1, "Writing bigsize"); + if (gzwrite(outf, buf, len) == 0) + err(1, "Writing bigsize"); } -static u64 read_bigsize(int infd) +static u64 read_bigsize(gzFile inf) { u64 val; u8 buf[BIGSIZE_MAX_LEN]; - if (!read_all(infd, buf, 1)) + if (gzread(inf, buf, 1) != 1) errx(1, "Reading bigsize"); switch (buf[0]) { case 0xfd: - if (!read_all(infd, buf+1, 2)) + if (gzread(inf, buf+1, 2) != 2) errx(1, "Reading bigsize"); break; case 0xfe: - if (!read_all(infd, buf+1, 4)) + if (gzread(inf, buf+1, 4) != 4) errx(1, "Reading bigsize"); break; case 0xff: - if (!read_all(infd, buf+1, 8)) + if (gzread(inf, buf+1, 8) != 8) errx(1, "Reading bigsize"); break; } @@ -153,7 +173,7 @@ static size_t find_index(const u64 *template, u64 val) /* All templates are of the same form. Output all the distinct values, then * write out which one is used by each channel */ -static void write_template_and_values(int outfd, const u64 *vals, const char *what) +static void write_template_and_values(gzFile outf, const u64 *vals, const char *what) { /* Sort and remove dups */ const u64 *template = deduplicate(tmpctx, vals); @@ -164,18 +184,18 @@ static void write_template_and_values(int outfd, const u64 *vals, const char *wh assert(tal_count(vals) >= tal_count(template)); /* Write template. */ - write_bigsize(outfd, tal_count(template)); + write_bigsize(outf, tal_count(template)); for (size_t i = 0; i < tal_count(template); i++) - write_bigsize(outfd, template[i]); + write_bigsize(outf, template[i]); /* Tie every channel into the template. O(N^2) but who * cares? */ for (size_t i = 0; i < tal_count(vals); i++) { - write_bigsize(outfd, find_index(template, vals[i])); + write_bigsize(outf, find_index(template, vals[i])); } } -static void write_bidir_perchan(int outfd, +static void write_bidir_perchan(gzFile outf, struct gossmap *gossmap, struct gossmap_chan **chans, u64 (*get_value)(struct gossmap *, @@ -194,7 +214,7 @@ static void write_bidir_perchan(int outfd, } } - write_template_and_values(outfd, vals, what); + write_template_and_values(outf, vals, what); } static u64 get_htlc_min(struct gossmap *gossmap, @@ -420,13 +440,13 @@ static void write_update(int outfd, write_msg_to_gstore(outfd, take(msg)); } -static const u64 *read_template(const tal_t *ctx, int infd, const char *what) +static const u64 *read_template(const tal_t *ctx, gzFile inf, const char *what) { - size_t count = read_bigsize(infd); + size_t count = read_bigsize(inf); u64 *template = tal_arr(ctx, u64, count); for (size_t i = 0; i < count; i++) - template[i] = read_bigsize(infd); + template[i] = read_bigsize(inf); if (verbose) printf("%zu unique %s\n", count, what); @@ -434,9 +454,9 @@ static const u64 *read_template(const tal_t *ctx, int infd, const char *what) return template; } -static u64 read_val(int infd, const u64 *template) +static u64 read_val(gzFile inf, const u64 *template) { - size_t idx = read_bigsize(infd); + size_t idx = read_bigsize(inf); assert(idx < tal_count(template)); return template[idx]; } @@ -478,6 +498,7 @@ int main(int argc, char *argv[]) size_t *node_to_compr_idx; size_t node_count, channel_count; struct gossmap_chan **chans, *c; + gzFile outf = gzdopen(outfd, "wb9"); struct gossmap *gossmap = gossmap_load_fd(tmpctx, infd, NULL, NULL, NULL); if (!gossmap) @@ -501,7 +522,7 @@ int main(int argc, char *argv[]) for (size_t i = 0; i < tal_count(nodes); i++) node_to_compr_idx[gossmap_node_idx(gossmap, nodes[i])] = i; - if (!write_all(outfd, GC_HEADER, GC_HEADERLEN)) + if (gzwrite(outf, GC_HEADER, GC_HEADERLEN) == 0) err(1, "Writing header"); /* Now, output channels. First get exact count. */ @@ -516,7 +537,7 @@ int main(int argc, char *argv[]) chans = tal_arr(gossmap, struct gossmap_chan *, channel_count); /* * := {channel_count} {start_nodeidx}*{channel_count} {end_nodeidx}*{channel_count} */ - write_bigsize(outfd, channel_count); + write_bigsize(outf, channel_count); size_t chanidx = 0; /* We iterate nodes to get to channels. This gives us nicer ordering for compression */ for (size_t wanted_dir = 0; wanted_dir < 2; wanted_dir++) { @@ -527,7 +548,7 @@ int main(int argc, char *argv[]) if (dir != wanted_dir) continue; - write_bigsize(outfd, + write_bigsize(outf, node_to_compr_idx[gossmap_node_idx(gossmap, n)]); /* First time reflects channel index for reader */ if (wanted_dir == 0) @@ -545,12 +566,12 @@ int main(int argc, char *argv[]) if (chans[i]->cupdate_off[dir] == 0) num_unknown++; if (!chans[i]->half[dir].enabled) { - write_bigsize(outfd, i * 2 + dir); + write_bigsize(outf, i * 2 + dir); num_disabled++; } } } - write_bigsize(outfd, channel_count * 2); + write_bigsize(outf, channel_count * 2); if (verbose) printf("%zu disabled channels (%zu no update)\n", num_disabled, num_unknown); @@ -562,7 +583,7 @@ int main(int argc, char *argv[]) gossmap_chan_get_capacity(gossmap, chans[i], &sats); vals[i] = sats.satoshis; /* Raw: compression format */ } - write_template_and_values(outfd, vals, "capacities"); + write_template_and_values(outf, vals, "capacities"); /* These are all of same form: one entry per direction per channel */ /* := {channel_count}*{htlc_min_idx} */ @@ -575,11 +596,12 @@ int main(int argc, char *argv[]) /* := {propfee_count} {propfee_count}*{propfee} */ /* := {channel_count}*{delay_idx} */ /* := {delay_count} {delay_count}*{delay} */ - write_bidir_perchan(outfd, gossmap, chans, get_htlc_min, "htlc_min"); - write_bidir_perchan(outfd, gossmap, chans, get_htlc_max, "htlc_max"); - write_bidir_perchan(outfd, gossmap, chans, get_basefee, "basefee"); - write_bidir_perchan(outfd, gossmap, chans, get_propfee, "propfee"); - write_bidir_perchan(outfd, gossmap, chans, get_delay, "delay"); + write_bidir_perchan(outf, gossmap, chans, get_htlc_min, "htlc_min"); + write_bidir_perchan(outf, gossmap, chans, get_htlc_max, "htlc_max"); + write_bidir_perchan(outf, gossmap, chans, get_basefee, "basefee"); + write_bidir_perchan(outf, gossmap, chans, get_propfee, "propfee"); + write_bidir_perchan(outf, gossmap, chans, get_delay, "delay"); + gzclose(outf); } else if (streq(argv[1], "decompress")) { char hdr[GC_HEADERLEN]; size_t channel_count, chanidx; @@ -596,19 +618,20 @@ int main(int argc, char *argv[]) } *chans; const u8 version = GOSSIP_STORE_VER; size_t disabled_count; + gzFile inf = gzdopen(infd, "rb"); - if (!read_all(infd, hdr, sizeof(hdr)) + if (gzread(inf, hdr, sizeof(hdr)) != sizeof(hdr) || !memeq(hdr, sizeof(hdr), GC_HEADER, GC_HEADERLEN)) errx(1, "Not a valid compressed gossmap header"); - channel_count = read_bigsize(infd); + channel_count = read_bigsize(inf); if (verbose) printf("%zu channels\n", channel_count); chans = tal_arrz(tmpctx, struct fakechan, channel_count); for (size_t i = 0; i < channel_count; i++) - chans[i].node1 = read_bigsize(infd); + chans[i].node1 = read_bigsize(inf); for (size_t i = 0; i < channel_count; i++) - chans[i].node2 = read_bigsize(infd); + chans[i].node2 = read_bigsize(inf); if (verbose >= 2) { for (size_t i = 0; i < channel_count; i++) { @@ -623,27 +646,27 @@ int main(int argc, char *argv[]) } disabled_count = 0; - while ((chanidx = read_bigsize(infd)) < channel_count*2) { + while ((chanidx = read_bigsize(inf)) < channel_count*2) { disabled_count++; chans[chanidx/2].half[chanidx%2].disabled = true; } if (verbose) printf("%zu disabled\n", disabled_count); - template = read_template(tmpctx, infd, "capacities"); + template = read_template(tmpctx, inf, "capacities"); for (size_t i = 0; i < channel_count; i++) - chans[i].capacity = read_val(infd, template); + chans[i].capacity = read_val(inf, template); - template = read_template(tmpctx, infd, "htlc_min"); + template = read_template(tmpctx, inf, "htlc_min"); for (size_t i = 0; i < channel_count; i++) { for (size_t dir = 0; dir < 2; dir++) { - chans[i].half[dir].htlc_min = read_val(infd, template); + chans[i].half[dir].htlc_min = read_val(inf, template); } } - template = read_template(tmpctx, infd, "htlc_max"); + template = read_template(tmpctx, inf, "htlc_max"); for (size_t i = 0; i < channel_count; i++) { for (size_t dir = 0; dir < 2; dir++) { - u64 v = read_val(infd, template); + u64 v = read_val(inf, template); if (v == 0) v = chans[i].capacity; else if (v == 1) @@ -651,27 +674,28 @@ int main(int argc, char *argv[]) chans[i].half[dir].htlc_max = v; } } - template = read_template(tmpctx, infd, "basefee"); + template = read_template(tmpctx, inf, "basefee"); for (size_t i = 0; i < channel_count; i++) { for (size_t dir = 0; dir < 2; dir++) { - chans[i].half[dir].basefee = read_val(infd, template); + chans[i].half[dir].basefee = read_val(inf, template); } } - template = read_template(tmpctx, infd, "propfee"); + template = read_template(tmpctx, inf, "propfee"); for (size_t i = 0; i < channel_count; i++) { for (size_t dir = 0; dir < 2; dir++) { - chans[i].half[dir].propfee = read_val(infd, template); + chans[i].half[dir].propfee = read_val(inf, template); } } - template = read_template(tmpctx, infd, "delay"); + template = read_template(tmpctx, inf, "delay"); for (size_t i = 0; i < channel_count; i++) { for (size_t dir = 0; dir < 2; dir++) { - chans[i].half[dir].delay = read_val(infd, template); + chans[i].half[dir].delay = read_val(inf, template); } } /* Now write out gossmap */ - write(outfd, &version, 1); + if (write(outfd, &version, 1) != 1) + err(1, "Failed to write output"); for (size_t i = 0; i < channel_count; i++) { write_announce(outfd, chans[i].node1, @@ -689,6 +713,7 @@ int main(int argc, char *argv[]) chans[i].half[dir].delay); } } + gzclose(inf); } else opt_usage_and_exit("Unknown command");