ci: Add a testpack.tar target to tranfer artifacts between CI jobs

The tester CI job uses absolute paths to ensure it is testing the
correct binaries. That clashes with the transfer between builder and
tester job using the `install` target because that switches things
around. This commit introduces a new target that just collects
artifacts in place, and tars them. Then we can use `tar` to unpack
them on the tester jobs again.

.github/workflows/ci.yaml

@@ -105,18 +105,14 @@ jobs:
           python3 -m pip install -r requirements.txt
           ./configure CC="$COMPILER"
 
-          # We'll need lowdown, not present on ubuntu:20.04 yet
+          make -j $(nproc) testpack.tar.bz2
-          make -j $(nproc) doc-all  # Builds lowdown as a dependency
-          make -j $(nproc) DESTDIR=${GITHUB_WORKSPACE}/cln-${{ matrix.CFG }} install
 
-          # The above leaves the binaries installed in
+          # Rename now so we don't clash
-          # cln-${matrix.CFG}, let's package it up and store it.
+          mv testpack.tar cln-${CFG}.tar
-
-          tar -cvzf /tmp/cln-${CFG}.tar.gz cln-${CFG}
       - uses: actions/upload-artifact@v3
         with:
-          name: cln-${{ matrix.CFG }}.tar.gz
+          name: cln-${{ matrix.CFG }}.tar
-          path: /tmp/cln-${{ matrix.CFG }}.tar.gz
+          path: cln-${{ matrix.CFG }}.tar
 
   integration:
     name: Test CLN ${{ matrix.cfg }}
@@ -172,7 +168,7 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v3
         with:
-          name: cln-${{ matrix.CFG }}.tar.gz
+          name: cln-${{ matrix.CFG }}.tar
 
       - name: Test
         env:
@@ -186,14 +182,10 @@ jobs:
           PYTEST_PAR: 10
         run: |
           set -e
+          tar -xvf cln-${CFG}.tar
+
           pip3 install --user pip wheel poetry
           poetry export -o requirements.txt --with dev --without-hashes
           python3 -m pip install -r requirements.txt
 
-          # Needed in order to run `make` below.
+          poetry run make pytest
-          ./configure CC="$COMPILER"
-
-          # Only run the `pytest` tests since these are the ones that
-          # really take time, and unit tests should be run in the
-          # `compile` step.
-          make pytest

Makefile

@@ -803,6 +803,27 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(
 
 install: install-program install-data
 
+# Non-artifacts that are needed for testing. These are added to the
+# testpack.tar, used to transfer things between builder and tester
+# phase. If you get a missing file/executable while testing on CI it
+# is likely missing from this variable.
+TESTBINS = \
+	target/${RUST_PROFILE}/examples/cln-rpc-getinfo \
+	target/${RUST_PROFILE}/examples/cln-plugin-startup \
+	tests/plugins/test_libplugin \
+	tests/plugins/test_selfdisable_after_getmanifest \
+	tools/hsmtool
+
+# The testpack is used in CI to transfer built artefacts between the
+# build and the test phase. This is necessary because the fixtures in
+# `tests/` explicitly use the binaries built in the current directory
+# rather than using `$PATH`, as that may pick up some other installed
+# version of `lightningd` leading to bogus results. We bundle up all
+# built artefacts here, and will unpack them on the tester (overlaying
+# on top of the checked out repo as if we had just built it in place).
+testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS)
+	tar -caf $@ $^
+
 uninstall:
 	@$(NORMAL_UNINSTALL)
 	@for f in $(BIN_PROGRAMS); do \