From daaf7b7c6bd6eaa2975777f1092dc01248c680fd Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 26 Sep 2025 12:18:24 +0000 Subject: [PATCH 1/4] appimage build: split out building type2-runtime into separate .sh --- contrib/build-linux/appimage/build.sh | 28 +------------- .../appimage/make_type2_runtime.sh | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 27 deletions(-) create mode 100755 contrib/build-linux/appimage/make_type2_runtime.sh diff --git a/contrib/build-linux/appimage/build.sh b/contrib/build-linux/appimage/build.sh index 8c1bfd7f5..c16b19f95 100755 --- a/contrib/build-linux/appimage/build.sh +++ b/contrib/build-linux/appimage/build.sh @@ -13,10 +13,6 @@ CONTRIB_APPIMAGE="$CONTRIB/build-linux/appimage" DISTDIR="$PROJECT_ROOT/dist" BUILD_UID=$(/usr/bin/stat -c %u "$PROJECT_ROOT") -# when bumping the runtime commit also check if the `type2-runtime-reproducible-build.patch` still works -TYPE2_RUNTIME_COMMIT="5e7217b7cfeecee1491c2d251e355c3cf8ba6e4d" -TYPE2_RUNTIME_REPO="https://github.com/AppImage/type2-runtime.git" - . "$CONTRIB"/build_tools_util.sh @@ -52,29 +48,7 @@ fi # build the type2-runtime binary, this build step uses a separate docker container # defined in the type2-runtime repo (patched with type2-runtime-reproducible-build.patch) -TYPE2_RUNTIME_REPO_DIR="$PROJECT_ROOT_OR_FRESHCLONE_ROOT/contrib/build-linux/appimage/.cache/appimage/type2-runtime" -( - if [ -f "$TYPE2_RUNTIME_REPO_DIR/runtime-x86_64" ]; then - info "type2-runtime already built, skipping" - exit 0 - fi - clone_or_update_repo "$TYPE2_RUNTIME_REPO" "$TYPE2_RUNTIME_COMMIT" "$TYPE2_RUNTIME_REPO_DIR" - - # Apply patch to make runtime build reproducible - info "Applying type2-runtime patch..." - cd "$TYPE2_RUNTIME_REPO_DIR" - git apply "$CONTRIB_APPIMAGE/patches/type2-runtime-reproducible-build.patch" || fail "Failed to apply runtime repo patch" - - info "building type2-runtime in build container..." - cd "$TYPE2_RUNTIME_REPO_DIR/scripts/docker" - env ARCH=x86_64 ./build-with-docker.sh - mv "./runtime-x86_64" "$TYPE2_RUNTIME_REPO_DIR/" - - # clean up the empty created 'out' dir to prevent permission issues - rm -rf "$TYPE2_RUNTIME_REPO_DIR/out" - - info "runtime build successful: $(sha256sum "$TYPE2_RUNTIME_REPO_DIR/runtime-x86_64")" -) +"$CONTRIB_APPIMAGE/make_type2_runtime.sh" || fail "Error building type2-runtime." info "building binary..." # check uid and maybe chown. see #8261 diff --git a/contrib/build-linux/appimage/make_type2_runtime.sh b/contrib/build-linux/appimage/make_type2_runtime.sh new file mode 100755 index 000000000..fdf751303 --- /dev/null +++ b/contrib/build-linux/appimage/make_type2_runtime.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.." +PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT" +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_APPIMAGE="$CONTRIB/build-linux/appimage" + +# when bumping the runtime commit also check if the `type2-runtime-reproducible-build.patch` still works +TYPE2_RUNTIME_COMMIT="5e7217b7cfeecee1491c2d251e355c3cf8ba6e4d" +TYPE2_RUNTIME_REPO="https://github.com/AppImage/type2-runtime.git" + +. "$CONTRIB"/build_tools_util.sh + + +TYPE2_RUNTIME_REPO_DIR="$PROJECT_ROOT_OR_FRESHCLONE_ROOT/contrib/build-linux/appimage/.cache/appimage/type2-runtime" +if [ -f "$TYPE2_RUNTIME_REPO_DIR/runtime-x86_64" ]; then + info "type2-runtime already built, skipping" + exit 0 +fi +clone_or_update_repo "$TYPE2_RUNTIME_REPO" "$TYPE2_RUNTIME_COMMIT" "$TYPE2_RUNTIME_REPO_DIR" + +# Apply patch to make runtime build reproducible +info "Applying type2-runtime patch..." +cd "$TYPE2_RUNTIME_REPO_DIR" +git apply "$CONTRIB_APPIMAGE/patches/type2-runtime-reproducible-build.patch" || fail "Failed to apply runtime repo patch" + +info "building type2-runtime in build container..." +cd "$TYPE2_RUNTIME_REPO_DIR/scripts/docker" +env ARCH=x86_64 ./build-with-docker.sh +mv "./runtime-x86_64" "$TYPE2_RUNTIME_REPO_DIR/" + +# clean up the empty created 'out' dir to prevent permission issues +rm -rf "$TYPE2_RUNTIME_REPO_DIR/out" + +info "runtime build successful: $(sha256sum "$TYPE2_RUNTIME_REPO_DIR/runtime-x86_64")" From b676c1e9d3fb173cc905eac53bfbbc2963cfe6f2 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 26 Sep 2025 12:27:32 +0000 Subject: [PATCH 2/4] build: docker run: make "-it" flags conditional on tty being available from https://stackoverflow.com/a/69088164 I am trying to run the build.sh scripts directly on a cirrus CI runner, and "docker run -it" is erroring due to lack of a tty. --- contrib/android/build.sh | 7 +++++-- contrib/build-linux/appimage/build.sh | 8 +++++++- contrib/build-linux/sdist/build.sh | 8 +++++++- contrib/build-wine/build.sh | 8 +++++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/contrib/android/build.sh b/contrib/android/build.sh index 08a9b31fb..a156838eb 100755 --- a/contrib/android/build.sh +++ b/contrib/android/build.sh @@ -63,11 +63,14 @@ else fi DOCKER_RUN_FLAGS="" - if [[ "$3" == "release" ]] ; then info "'release' mode selected. mounting ~/.keystore inside container." DOCKER_RUN_FLAGS="-v $HOME/.keystore:/home/user/.keystore" fi +if sh -c ": >/dev/tty" >/dev/null 2>/dev/null; then + info "/dev/tty is available and usable" + DOCKER_RUN_FLAGS="$DOCKER_RUN_FLAGS -it" +fi info "building binary..." mkdir --parents "$PROJECT_ROOT_OR_FRESHCLONE_ROOT"/.buildozer/.gradle @@ -78,7 +81,7 @@ if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) sudo chown -R 1000:1000 "$FRESH_CLONE" fi fi -docker run -it --rm \ +docker run --rm \ --name electrum-android-builder-cont \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/home/user/wspace/electrum \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT"/.buildozer/.gradle:/home/user/.gradle \ diff --git a/contrib/build-linux/appimage/build.sh b/contrib/build-linux/appimage/build.sh index c16b19f95..8f66d5359 100755 --- a/contrib/build-linux/appimage/build.sh +++ b/contrib/build-linux/appimage/build.sh @@ -50,6 +50,12 @@ fi # defined in the type2-runtime repo (patched with type2-runtime-reproducible-build.patch) "$CONTRIB_APPIMAGE/make_type2_runtime.sh" || fail "Error building type2-runtime." +DOCKER_RUN_FLAGS="" +if sh -c ": >/dev/tty" >/dev/null 2>/dev/null; then + info "/dev/tty is available and usable" + DOCKER_RUN_FLAGS="-it" +fi + info "building binary..." # check uid and maybe chown. see #8261 if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) @@ -58,7 +64,7 @@ if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) sudo chown -R 1000:1000 "$FRESH_CLONE" fi fi -docker run -it \ +docker run $DOCKER_RUN_FLAGS \ --name electrum-appimage-builder-cont \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \ --rm \ diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh index 11a746273..aa68e08f6 100755 --- a/contrib/build-linux/sdist/build.sh +++ b/contrib/build-linux/sdist/build.sh @@ -46,6 +46,12 @@ else info "not doing fresh clone." fi +DOCKER_RUN_FLAGS="" +if sh -c ": >/dev/tty" >/dev/null 2>/dev/null; then + info "/dev/tty is available and usable" + DOCKER_RUN_FLAGS="-it" +fi + info "building binary..." # check uid and maybe chown. see #8261 if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) @@ -54,7 +60,7 @@ if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) sudo chown -R 1000:1000 "$FRESH_CLONE" fi fi -docker run -it \ +docker run $DOCKER_RUN_FLAGS \ --name electrum-sdist-builder-cont \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \ --rm \ diff --git a/contrib/build-wine/build.sh b/contrib/build-wine/build.sh index f3bc4e6a9..c10e8751a 100755 --- a/contrib/build-wine/build.sh +++ b/contrib/build-wine/build.sh @@ -48,6 +48,12 @@ else info "not doing fresh clone." fi +DOCKER_RUN_FLAGS="" +if sh -c ": >/dev/tty" >/dev/null 2>/dev/null; then + info "/dev/tty is available and usable" + DOCKER_RUN_FLAGS="-it" +fi + info "building binary..." # check uid and maybe chown. see #8261 if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) @@ -56,7 +62,7 @@ if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) sudo chown -R 1000:1000 "$FRESH_CLONE" fi fi -docker run -it \ +docker run $DOCKER_RUN_FLAGS \ --name electrum-wine-builder-cont \ -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/wine64/drive_c/electrum \ --rm \ From 447c7cf8b3c9649ba9cca398a6fadf8f5525a2c7 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 26 Sep 2025 12:37:50 +0000 Subject: [PATCH 3/4] build: Dockerfile: mod "new user to avoid using root" to support UID=0 Extend "create new user to avoid using root" copypasta to support being run as root (ARG UID=0). When running build.sh directly on a cirrus CI runner, we are running as root. --- contrib/android/Dockerfile | 15 +++++++-------- contrib/build-linux/appimage/Dockerfile | 13 ++++++------- contrib/build-linux/sdist/Dockerfile | 13 ++++++------- contrib/build-wine/Dockerfile | 15 +++++++-------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/contrib/android/Dockerfile b/contrib/android/Dockerfile index d1f1c2104..74f53928e 100644 --- a/contrib/android/Dockerfile +++ b/contrib/android/Dockerfile @@ -174,17 +174,16 @@ RUN apt -y update -qq \ # create new user to avoid using root; but with sudo access and no password for convenience. ARG UID=1000 -ENV USER="user" -ENV HOME_DIR="/home/${USER}" +RUN if [ "$UID" != "0" ] ; then useradd --uid $UID --create-home --shell /bin/bash "user" ; fi +RUN usermod -append --groups sudo $(id -nu $UID || echo "user") +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN HOME_DIR=$(getent passwd $UID | cut -d: -f6) ENV WORK_DIR="${HOME_DIR}/wspace" \ PATH="${HOME_DIR}/.local/bin:${PATH}" -RUN useradd --uid $UID --create-home --shell /bin/bash ${USER} -RUN usermod -append --groups sudo ${USER} -RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers WORKDIR ${WORK_DIR} -RUN chown --recursive ${USER} ${WORK_DIR} ${ANDROID_SDK_HOME} -RUN chown ${USER} /opt -USER ${USER} +RUN chown --recursive ${UID} ${WORK_DIR} ${ANDROID_SDK_HOME} +RUN chown ${UID} /opt +USER ${UID} # venv, VIRTUAL_ENV is used by buildozer to indicate a venv environment ENV VIRTUAL_ENV=/opt/venv diff --git a/contrib/build-linux/appimage/Dockerfile b/contrib/build-linux/appimage/Dockerfile index 2b5148c64..5f43ac930 100644 --- a/contrib/build-linux/appimage/Dockerfile +++ b/contrib/build-linux/appimage/Dockerfile @@ -75,13 +75,12 @@ RUN apt-get update -q && \ # create new user to avoid using root; but with sudo access and no password for convenience. ARG UID=1000 -ENV USER="user" -ENV HOME_DIR="/home/${USER}" +RUN if [ "$UID" != "0" ] ; then useradd --uid $UID --create-home --shell /bin/bash "user" ; fi +RUN usermod -append --groups sudo $(id -nu $UID || echo "user") +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN HOME_DIR=$(getent passwd $UID | cut -d: -f6) ENV WORK_DIR="${HOME_DIR}/wspace" \ PATH="${HOME_DIR}/.local/bin:${PATH}" -RUN useradd --uid $UID --create-home --shell /bin/bash ${USER} -RUN usermod -append --groups sudo ${USER} -RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers WORKDIR ${WORK_DIR} -RUN chown --recursive ${USER} ${WORK_DIR} -USER ${USER} +RUN chown --recursive ${UID} ${WORK_DIR} +USER ${UID} diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile index d397e85a7..3b6f12112 100644 --- a/contrib/build-linux/sdist/Dockerfile +++ b/contrib/build-linux/sdist/Dockerfile @@ -18,13 +18,12 @@ RUN apt-get update -q && \ # create new user to avoid using root; but with sudo access and no password for convenience. ARG UID=1000 -ENV USER="user" -ENV HOME_DIR="/home/${USER}" +RUN if [ "$UID" != "0" ] ; then useradd --uid $UID --create-home --shell /bin/bash "user" ; fi +RUN usermod -append --groups sudo $(id -nu $UID || echo "user") +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN HOME_DIR=$(getent passwd $UID | cut -d: -f6) ENV WORK_DIR="${HOME_DIR}/wspace" \ PATH="${HOME_DIR}/.local/bin:${PATH}" -RUN useradd --uid $UID --create-home --shell /bin/bash ${USER} -RUN usermod -append --groups sudo ${USER} -RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers WORKDIR ${WORK_DIR} -RUN chown --recursive ${USER} ${WORK_DIR} -USER ${USER} +RUN chown --recursive ${UID} ${WORK_DIR} +USER ${UID} diff --git a/contrib/build-wine/Dockerfile b/contrib/build-wine/Dockerfile index f62b1da1d..4a7022bce 100644 --- a/contrib/build-wine/Dockerfile +++ b/contrib/build-wine/Dockerfile @@ -58,16 +58,15 @@ RUN DEBIAN_CODENAME=$(lsb_release --codename --short) && \ # create new user to avoid using root; but with sudo access and no password for convenience. ARG UID=1000 -ENV USER="user" -ENV HOME_DIR="/home/${USER}" +RUN if [ "$UID" != "0" ] ; then useradd --uid $UID --create-home --shell /bin/bash "user" ; fi +RUN usermod -append --groups sudo $(id -nu $UID || echo "user") +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN HOME_DIR=$(getent passwd $UID | cut -d: -f6) ENV WORK_DIR="${HOME_DIR}/wspace" \ PATH="${HOME_DIR}/.local/bin:${PATH}" -RUN useradd --uid $UID --create-home --shell /bin/bash ${USER} -RUN usermod -append --groups sudo ${USER} -RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers WORKDIR ${WORK_DIR} -RUN chown --recursive ${USER} ${WORK_DIR} -RUN chown ${USER} /opt -USER ${USER} +RUN chown --recursive ${UID} ${WORK_DIR} +RUN chown ${UID} /opt +USER ${UID} RUN mkdir --parents "/opt/wine64/drive_c/electrum" From 750f1d0c9953365ce2ff65c487e6fe54825c07e9 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 26 Sep 2025 12:42:37 +0000 Subject: [PATCH 4/4] ci: appimage build: run build.sh directly instead of sub-scripts - previously we had the CI spawn us inside a docker container as per our Dockerfile, and we ran make_appimage.sh inside that - now we also need to run make_type2_runtime.sh, which builds and runs another docker container, so the old approach does not work - follow-up https://github.com/spesmilo/electrum/pull/10019 - looks easier to just try to run build.sh, the outer script, which does all that - makes the CI task more similar to what dev machines run - at the cost of somewhat more compute --- .cirrus.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index c1a3529ff..13c2504eb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -344,8 +344,10 @@ task: only_if: $CIRRUS_CRON == "" - trigger_type: automatic only_if: $CIRRUS_CRON == "nightly" - container: - dockerfile: contrib/build-linux/appimage/Dockerfile + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux cpu: 2 memory: 2G pip_cache: @@ -361,11 +363,9 @@ task: - cat contrib/make_libsecp256k1.sh | sha256sum - git ls-files -s contrib/build-linux/appimage/ build_script: - - ./contrib/build-linux/appimage/make_appimage.sh + - ./contrib/build-linux/appimage/build.sh binaries_artifacts: path: "dist/*" - env: - CIRRUS_DOCKER_CONTEXT: contrib/build-linux/appimage depends_on: - "unittests: py3.10"