davide/palladum-lightning
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

devtools/gossmap-compress: use transparent zlib compression if available.

Browse Source 
Before:
```
-rw-rw-r-- 1 rusty rusty 1643258 Jul 26 09:51 compressed
```

After:
```
-rw-rw-r-- 1 rusty rusty 508332 Jul 26 09:49 compressed
```

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
Rusty Russell
2024-08-07 10:15:55 +09:30
parent c93b4aafb2
commit 99e8e9246e
4 changed files with 94 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@@ -658,7 +658,7 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o
# uses some ccan modules internally). We want to rely on -lwallycore etc. # uses some ccan modules internally). We want to rely on -lwallycore etc.
# (as per EXTERNAL_LDLIBS) so we filter them out here. # (as per EXTERNAL_LDLIBS) so we filter them out here.
$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS): $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
@$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a -o $@) @$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a $($(@)_LDLIBS) -o $@)
# We special case the fuzzing target binaries, as they need to link against libfuzzer, # We special case the fuzzing target binaries, as they need to link against libfuzzer,
# which brings its own main(). # which brings its own main().

14
configure vendored
View File

@@ -367,6 +367,20 @@ trap "rm -f $CONFIG_VAR_FILE.$$" 0
$CONFIGURATOR --extra-tests --autotools-style --var-file=$CONFIG_VAR_FILE.$$ --header-file=$CONFIG_HEADER.$$ --configurator-cc="$CONFIGURATOR_CC" --wrapper="$CONFIGURATOR_WRAPPER" "$CC" ${CWARNFLAGS-$BASE_WARNFLAGS} $CDEBUGFLAGS $COPTFLAGS $CSANFLAGS -I$CPATH -L$LIBRARY_PATH $SQLITE3_CFLAGS $POSTGRES_INCLUDE <<EOF $CONFIGURATOR --extra-tests --autotools-style --var-file=$CONFIG_VAR_FILE.$$ --header-file=$CONFIG_HEADER.$$ --configurator-cc="$CONFIGURATOR_CC" --wrapper="$CONFIGURATOR_WRAPPER" "$CC" ${CWARNFLAGS-$BASE_WARNFLAGS} $CDEBUGFLAGS $COPTFLAGS $CSANFLAGS -I$CPATH -L$LIBRARY_PATH $SQLITE3_CFLAGS $POSTGRES_INCLUDE <<EOF
var=HAVE_ZLIB
desc=zlib support
style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE
link=-lz
code=
#include <zlib.h>
#include <stdlib.h>
int main(void)
{
gzFile f = gzopen("/dev/null", "wb");
return f != NULL ? 0 : 1;
}
/*END*/
var=HAVE_GOOD_LIBSODIUM var=HAVE_GOOD_LIBSODIUM
desc=libsodium with IETF chacha20 variants desc=libsodium with IETF chacha20 variants
style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE style=DEFINES_EVERYTHING|EXECUTE|MAY_NOT_COMPILE

5
devtools/Makefile
View File

@@ -11,6 +11,11 @@ ALL_C_SOURCES += $(DEVTOOLS_TOOL_SRC)
ALL_C_HEADERS += ALL_C_HEADERS +=
ALL_PROGRAMS += $(DEVTOOLS) ALL_PROGRAMS += $(DEVTOOLS)
# gossmap-compress wants -lz if we say we have it.
ifeq ($(HAVE_ZLIB),1)
devtools/gossmap-compress_LDLIBS=-lz
endif # ZLIB
DEVTOOLS_COMMON_OBJS := \ DEVTOOLS_COMMON_OBJS := \
common/amount.o \ common/amount.o \
common/autodata.o \ common/autodata.o \

123
devtools/gossmap-compress.c
View File

@@ -7,7 +7,6 @@
#include <ccan/mem/mem.h> #include <ccan/mem/mem.h>
#include <ccan/opt/opt.h> #include <ccan/opt/opt.h>
#include <ccan/read_write_all/read_write_all.h> #include <ccan/read_write_all/read_write_all.h>
#include <ccan/tal/grab_file/grab_file.h>
#include <ccan/tal/str/str.h> #include <ccan/tal/str/str.h>
#include <common/bigsize.h> #include <common/bigsize.h>
#include <common/gossip_store.h> #include <common/gossip_store.h>
@@ -16,8 +15,29 @@
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <gossipd/gossip_store_wiregen.h> #include <gossipd/gossip_store_wiregen.h>
#include <stdio.h>
#include <unistd.h> #include <unistd.h>
#include <wire/peer_wiregen.h> #include <wire/peer_wiregen.h>
#if HAVE_ZLIB
#include <zlib.h>
#else
/* Worst... zlib... ever! */
#define gzFile int
#define gzdopen(fd, mode) (fd)
#define gzclose(outf) close(outf)
static int gzread(int fd, void *buf, size_t len)
{
if (read_all(fd, buf, len))
return len;
return 0;
}
static int gzwrite(int fd, const void *buf, size_t len)
{
if (write_all(fd, buf, len))
return len;
return 0;
}
#endif
static unsigned int verbose = 0; static unsigned int verbose = 0;
@@ -70,35 +90,35 @@ static int cmp_node_num_chans(struct gossmap_node *const *a,
return (int)(*a)->num_chans - (int)(*b)->num_chans; return (int)(*a)->num_chans - (int)(*b)->num_chans;
} }
static void write_bigsize(int outfd, u64 val) static void write_bigsize(gzFile outf, u64 val)
{ {
u8 buf[BIGSIZE_MAX_LEN]; u8 buf[BIGSIZE_MAX_LEN];
size_t len; size_t len;
len = bigsize_put(buf, val); len = bigsize_put(buf, val);
if (!write_all(outfd, buf, len)) if (gzwrite(outf, buf, len) == 0)
errx(1, "Writing bigsize"); err(1, "Writing bigsize");
} }
static u64 read_bigsize(int infd) static u64 read_bigsize(gzFile inf)
{ {
u64 val; u64 val;
u8 buf[BIGSIZE_MAX_LEN]; u8 buf[BIGSIZE_MAX_LEN];
if (!read_all(infd, buf, 1)) if (gzread(inf, buf, 1) != 1)
errx(1, "Reading bigsize"); errx(1, "Reading bigsize");
switch (buf[0]) { switch (buf[0]) {
case 0xfd: case 0xfd:
if (!read_all(infd, buf+1, 2)) if (gzread(inf, buf+1, 2) != 2)
errx(1, "Reading bigsize"); errx(1, "Reading bigsize");
break; break;
case 0xfe: case 0xfe:
if (!read_all(infd, buf+1, 4)) if (gzread(inf, buf+1, 4) != 4)
errx(1, "Reading bigsize"); errx(1, "Reading bigsize");
break; break;
case 0xff: case 0xff:
if (!read_all(infd, buf+1, 8)) if (gzread(inf, buf+1, 8) != 8)
errx(1, "Reading bigsize"); errx(1, "Reading bigsize");
break; break;
} }
@@ -153,7 +173,7 @@ static size_t find_index(const u64 *template, u64 val)
/* All templates are of the same form. Output all the distinct values, then /* All templates are of the same form. Output all the distinct values, then
* write out which one is used by each channel */ * write out which one is used by each channel */
static void write_template_and_values(int outfd, const u64 *vals, const char *what) static void write_template_and_values(gzFile outf, const u64 *vals, const char *what)
{ {
/* Sort and remove dups */ /* Sort and remove dups */
const u64 *template = deduplicate(tmpctx, vals); const u64 *template = deduplicate(tmpctx, vals);
@@ -164,18 +184,18 @@ static void write_template_and_values(int outfd, const u64 *vals, const char *wh
assert(tal_count(vals) >= tal_count(template)); assert(tal_count(vals) >= tal_count(template));
/* Write template. */ /* Write template. */
write_bigsize(outfd, tal_count(template)); write_bigsize(outf, tal_count(template));
for (size_t i = 0; i < tal_count(template); i++) for (size_t i = 0; i < tal_count(template); i++)
write_bigsize(outfd, template[i]); write_bigsize(outf, template[i]);
/* Tie every channel into the template. O(N^2) but who /* Tie every channel into the template. O(N^2) but who
* cares? */ * cares? */
for (size_t i = 0; i < tal_count(vals); i++) { for (size_t i = 0; i < tal_count(vals); i++) {
write_bigsize(outfd, find_index(template, vals[i])); write_bigsize(outf, find_index(template, vals[i]));
} }
} }
static void write_bidir_perchan(int outfd, static void write_bidir_perchan(gzFile outf,
struct gossmap *gossmap, struct gossmap *gossmap,
struct gossmap_chan **chans, struct gossmap_chan **chans,
u64 (*get_value)(struct gossmap *, u64 (*get_value)(struct gossmap *,
@@ -194,7 +214,7 @@ static void write_bidir_perchan(int outfd,
} }
} }
write_template_and_values(outfd, vals, what); write_template_and_values(outf, vals, what);
} }
static u64 get_htlc_min(struct gossmap *gossmap, static u64 get_htlc_min(struct gossmap *gossmap,
@@ -420,13 +440,13 @@ static void write_update(int outfd,
write_msg_to_gstore(outfd, take(msg)); write_msg_to_gstore(outfd, take(msg));
} }
static const u64 *read_template(const tal_t *ctx, int infd, const char *what) static const u64 *read_template(const tal_t *ctx, gzFile inf, const char *what)
{ {
size_t count = read_bigsize(infd); size_t count = read_bigsize(inf);
u64 *template = tal_arr(ctx, u64, count); u64 *template = tal_arr(ctx, u64, count);
for (size_t i = 0; i < count; i++) for (size_t i = 0; i < count; i++)
template[i] = read_bigsize(infd); template[i] = read_bigsize(inf);
if (verbose) if (verbose)
printf("%zu unique %s\n", count, what); printf("%zu unique %s\n", count, what);
@@ -434,9 +454,9 @@ static const u64 *read_template(const tal_t *ctx, int infd, const char *what)
return template; return template;
} }
static u64 read_val(int infd, const u64 *template) static u64 read_val(gzFile inf, const u64 *template)
{ {
size_t idx = read_bigsize(infd); size_t idx = read_bigsize(inf);
assert(idx < tal_count(template)); assert(idx < tal_count(template));
return template[idx]; return template[idx];
} }
@@ -478,6 +498,7 @@ int main(int argc, char *argv[])
size_t *node_to_compr_idx; size_t *node_to_compr_idx;
size_t node_count, channel_count; size_t node_count, channel_count;
struct gossmap_chan **chans, *c; struct gossmap_chan **chans, *c;
gzFile outf = gzdopen(outfd, "wb9");
struct gossmap *gossmap = gossmap_load_fd(tmpctx, infd, NULL, NULL, NULL); struct gossmap *gossmap = gossmap_load_fd(tmpctx, infd, NULL, NULL, NULL);
if (!gossmap) if (!gossmap)
@@ -501,7 +522,7 @@ int main(int argc, char *argv[])
for (size_t i = 0; i < tal_count(nodes); i++) for (size_t i = 0; i < tal_count(nodes); i++)
node_to_compr_idx[gossmap_node_idx(gossmap, nodes[i])] = i; node_to_compr_idx[gossmap_node_idx(gossmap, nodes[i])] = i;
if (!write_all(outfd, GC_HEADER, GC_HEADERLEN)) if (gzwrite(outf, GC_HEADER, GC_HEADERLEN) == 0)
err(1, "Writing header"); err(1, "Writing header");
/* Now, output channels. First get exact count. */ /* Now, output channels. First get exact count. */
@@ -516,7 +537,7 @@ int main(int argc, char *argv[])
chans = tal_arr(gossmap, struct gossmap_chan *, channel_count); chans = tal_arr(gossmap, struct gossmap_chan *, channel_count);
/* * <CHANNEL_ENDS> := {channel_count} {start_nodeidx}*{channel_count} {end_nodeidx}*{channel_count} */ /* * <CHANNEL_ENDS> := {channel_count} {start_nodeidx}*{channel_count} {end_nodeidx}*{channel_count} */
write_bigsize(outfd, channel_count); write_bigsize(outf, channel_count);
size_t chanidx = 0; size_t chanidx = 0;
/* We iterate nodes to get to channels. This gives us nicer ordering for compression */ /* We iterate nodes to get to channels. This gives us nicer ordering for compression */
for (size_t wanted_dir = 0; wanted_dir < 2; wanted_dir++) { for (size_t wanted_dir = 0; wanted_dir < 2; wanted_dir++) {
@@ -527,7 +548,7 @@ int main(int argc, char *argv[])
if (dir != wanted_dir) if (dir != wanted_dir)
continue; continue;
write_bigsize(outfd, write_bigsize(outf,
node_to_compr_idx[gossmap_node_idx(gossmap, n)]); node_to_compr_idx[gossmap_node_idx(gossmap, n)]);
/* First time reflects channel index for reader */ /* First time reflects channel index for reader */
if (wanted_dir == 0) if (wanted_dir == 0)
@@ -545,12 +566,12 @@ int main(int argc, char *argv[])
if (chans[i]->cupdate_off[dir] == 0) if (chans[i]->cupdate_off[dir] == 0)
num_unknown++; num_unknown++;
if (!chans[i]->half[dir].enabled) { if (!chans[i]->half[dir].enabled) {
write_bigsize(outfd, i * 2 + dir); write_bigsize(outf, i * 2 + dir);
num_disabled++; num_disabled++;
} }
} }
} }
write_bigsize(outfd, channel_count * 2); write_bigsize(outf, channel_count * 2);
if (verbose) if (verbose)
printf("%zu disabled channels (%zu no update)\n", num_disabled, num_unknown); printf("%zu disabled channels (%zu no update)\n", num_disabled, num_unknown);
@@ -562,7 +583,7 @@ int main(int argc, char *argv[])
gossmap_chan_get_capacity(gossmap, chans[i], &sats); gossmap_chan_get_capacity(gossmap, chans[i], &sats);
vals[i] = sats.satoshis; /* Raw: compression format */ vals[i] = sats.satoshis; /* Raw: compression format */
} }
write_template_and_values(outfd, vals, "capacities"); write_template_and_values(outf, vals, "capacities");
/* These are all of same form: one entry per direction per channel */ /* These are all of same form: one entry per direction per channel */
/* <HTLC_MINS> := <HTLC_MIN_TEMPLATES> {channel_count}*{htlc_min_idx} */ /* <HTLC_MINS> := <HTLC_MIN_TEMPLATES> {channel_count}*{htlc_min_idx} */
@@ -575,11 +596,12 @@ int main(int argc, char *argv[])
/* <PROPFEE_TEMPLATES> := {propfee_count} {propfee_count}*{propfee} */ /* <PROPFEE_TEMPLATES> := {propfee_count} {propfee_count}*{propfee} */
/* <DELAYS> := <DELAY_TEMPLATES> {channel_count}*{delay_idx} */ /* <DELAYS> := <DELAY_TEMPLATES> {channel_count}*{delay_idx} */
/* <DELAY_TEMPLATES> := {delay_count} {delay_count}*{delay} */ /* <DELAY_TEMPLATES> := {delay_count} {delay_count}*{delay} */
write_bidir_perchan(outfd, gossmap, chans, get_htlc_min, "htlc_min"); write_bidir_perchan(outf, gossmap, chans, get_htlc_min, "htlc_min");
write_bidir_perchan(outfd, gossmap, chans, get_htlc_max, "htlc_max"); write_bidir_perchan(outf, gossmap, chans, get_htlc_max, "htlc_max");
write_bidir_perchan(outfd, gossmap, chans, get_basefee, "basefee"); write_bidir_perchan(outf, gossmap, chans, get_basefee, "basefee");
write_bidir_perchan(outfd, gossmap, chans, get_propfee, "propfee"); write_bidir_perchan(outf, gossmap, chans, get_propfee, "propfee");
write_bidir_perchan(outfd, gossmap, chans, get_delay, "delay"); write_bidir_perchan(outf, gossmap, chans, get_delay, "delay");
gzclose(outf);
} else if (streq(argv[1], "decompress")) { } else if (streq(argv[1], "decompress")) {
char hdr[GC_HEADERLEN]; char hdr[GC_HEADERLEN];
size_t channel_count, chanidx; size_t channel_count, chanidx;
@@ -596,19 +618,20 @@ int main(int argc, char *argv[])
} *chans; } *chans;
const u8 version = GOSSIP_STORE_VER; const u8 version = GOSSIP_STORE_VER;
size_t disabled_count; size_t disabled_count;
gzFile inf = gzdopen(infd, "rb");
if (!read_all(infd, hdr, sizeof(hdr)) if (gzread(inf, hdr, sizeof(hdr)) != sizeof(hdr)
|| !memeq(hdr, sizeof(hdr), GC_HEADER, GC_HEADERLEN)) || !memeq(hdr, sizeof(hdr), GC_HEADER, GC_HEADERLEN))
errx(1, "Not a valid compressed gossmap header"); errx(1, "Not a valid compressed gossmap header");
channel_count = read_bigsize(infd); channel_count = read_bigsize(inf);
if (verbose) if (verbose)
printf("%zu channels\n", channel_count); printf("%zu channels\n", channel_count);
chans = tal_arrz(tmpctx, struct fakechan, channel_count); chans = tal_arrz(tmpctx, struct fakechan, channel_count);
for (size_t i = 0; i < channel_count; i++) for (size_t i = 0; i < channel_count; i++)
chans[i].node1 = read_bigsize(infd); chans[i].node1 = read_bigsize(inf);
for (size_t i = 0; i < channel_count; i++) for (size_t i = 0; i < channel_count; i++)
chans[i].node2 = read_bigsize(infd); chans[i].node2 = read_bigsize(inf);
if (verbose >= 2) { if (verbose >= 2) {
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
@@ -623,27 +646,27 @@ int main(int argc, char *argv[])
} }
disabled_count = 0; disabled_count = 0;
while ((chanidx = read_bigsize(infd)) < channel_count*2) { while ((chanidx = read_bigsize(inf)) < channel_count*2) {
disabled_count++; disabled_count++;
chans[chanidx/2].half[chanidx%2].disabled = true; chans[chanidx/2].half[chanidx%2].disabled = true;
} }
if (verbose) if (verbose)
printf("%zu disabled\n", disabled_count); printf("%zu disabled\n", disabled_count);
template = read_template(tmpctx, infd, "capacities"); template = read_template(tmpctx, inf, "capacities");
for (size_t i = 0; i < channel_count; i++) for (size_t i = 0; i < channel_count; i++)
chans[i].capacity = read_val(infd, template); chans[i].capacity = read_val(inf, template);
template = read_template(tmpctx, infd, "htlc_min"); template = read_template(tmpctx, inf, "htlc_min");
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
for (size_t dir = 0; dir < 2; dir++) { for (size_t dir = 0; dir < 2; dir++) {
chans[i].half[dir].htlc_min = read_val(infd, template); chans[i].half[dir].htlc_min = read_val(inf, template);
} }
} }
template = read_template(tmpctx, infd, "htlc_max"); template = read_template(tmpctx, inf, "htlc_max");
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
for (size_t dir = 0; dir < 2; dir++) { for (size_t dir = 0; dir < 2; dir++) {
u64 v = read_val(infd, template); u64 v = read_val(inf, template);
if (v == 0) if (v == 0)
v = chans[i].capacity; v = chans[i].capacity;
else if (v == 1) else if (v == 1)
@@ -651,27 +674,28 @@ int main(int argc, char *argv[])
chans[i].half[dir].htlc_max = v; chans[i].half[dir].htlc_max = v;
} }
} }
template = read_template(tmpctx, infd, "basefee"); template = read_template(tmpctx, inf, "basefee");
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
for (size_t dir = 0; dir < 2; dir++) { for (size_t dir = 0; dir < 2; dir++) {
chans[i].half[dir].basefee = read_val(infd, template); chans[i].half[dir].basefee = read_val(inf, template);
} }
} }
template = read_template(tmpctx, infd, "propfee"); template = read_template(tmpctx, inf, "propfee");
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
for (size_t dir = 0; dir < 2; dir++) { for (size_t dir = 0; dir < 2; dir++) {
chans[i].half[dir].propfee = read_val(infd, template); chans[i].half[dir].propfee = read_val(inf, template);
} }
} }
template = read_template(tmpctx, infd, "delay"); template = read_template(tmpctx, inf, "delay");
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
for (size_t dir = 0; dir < 2; dir++) { for (size_t dir = 0; dir < 2; dir++) {
chans[i].half[dir].delay = read_val(infd, template); chans[i].half[dir].delay = read_val(inf, template);
} }
} }
/* Now write out gossmap */ /* Now write out gossmap */
write(outfd, &version, 1); if (write(outfd, &version, 1) != 1)
err(1, "Failed to write output");
for (size_t i = 0; i < channel_count; i++) { for (size_t i = 0; i < channel_count; i++) {
write_announce(outfd, write_announce(outfd,
chans[i].node1, chans[i].node1,
@@ -689,6 +713,7 @@ int main(int argc, char *argv[])
chans[i].half[dir].delay); chans[i].half[dir].delay);
} }
} }
gzclose(inf);
} else } else
opt_usage_and_exit("Unknown command"); opt_usage_and_exit("Unknown command");