From bc763971efa866d4cf3cc92c45d7486c9eca4e68 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 30 Aug 2023 19:58:14 +0200 Subject: [PATCH 1/5] ci: Remove custom build step names Putting build matrix details into a build step name is rather useless as the jobs themselves already contain the needed information. --- .github/workflows/build_test.yml | 2 +- .github/workflows/cflite_pr.yml | 4 ++-- .github/workflows/cifuzz.yml | 4 ++-- .github/workflows/mkosi.yml | 8 ++++---- .github/workflows/unit_tests.yml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 16dcd30c082..f8b0ccaf81d 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -34,5 +34,5 @@ jobs: steps: - name: Repository checkout uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 - - name: ${{ format('Build check ({0}-{1}-{2}-{3})', env.COMPILER, env.COMPILER_VERSION, env.LINKER, env.CRYPTOLIB) }} + - name: Build check run: sudo -E .github/workflows/build_test.sh diff --git a/.github/workflows/cflite_pr.yml b/.github/workflows/cflite_pr.yml index a35a97f046d..707ea0b6ba6 100644 --- a/.github/workflows/cflite_pr.yml +++ b/.github/workflows/cflite_pr.yml @@ -23,13 +23,13 @@ jobs: matrix: sanitizer: [address, undefined, memory] steps: - - name: Build Fuzzers (${{ matrix.sanitizer }}) + - name: Build Fuzzers id: build uses: google/clusterfuzzlite/actions/build_fuzzers@v1 with: sanitizer: ${{ matrix.sanitizer }} github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Run Fuzzers (${{ matrix.sanitizer }}) + - name: Run Fuzzers id: run uses: google/clusterfuzzlite/actions/run_fuzzers@v1 with: diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index a7e4d2b8420..f7530b75072 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -39,7 +39,7 @@ jobs: security-events: write steps: - - name: Build Fuzzers (${{ matrix.sanitizer }}) + - name: Build Fuzzers id: build uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master with: @@ -51,7 +51,7 @@ jobs: sanitizer: ${{ matrix.sanitizer }} architecture: ${{ matrix.architecture }} output-sarif: true - - name: Run Fuzzers (${{ matrix.sanitizer }}) + - name: Run Fuzzers uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'systemd' diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index c353423c582..5b70c607c26 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -140,14 +140,14 @@ jobs: - name: Generate secure boot key run: mkosi --debug genkey - - name: Show ${{ matrix.distro }} image summary + - name: Show image summary run: mkosi summary - - name: Build ${{ matrix.distro }} + - name: Build run: mkosi --debug - - name: Boot ${{ matrix.distro }} systemd-nspawn + - name: Boot systemd-nspawn run: test "$(sudo mkosi --debug boot 1>&2; echo $?)" -eq 123 - - name: Boot ${{ matrix.distro }} QEMU + - name: Boot QEMU run: timeout -k 30 10m test "$(mkosi --debug qemu 1>&2; echo $?)" -eq 123 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 051e8ec3bf6..e2dd8c3da25 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -39,7 +39,7 @@ jobs: # Pass only specific env variables through sudo, to avoid having # the already existing XDG_* stuff on the "other side" sudo --preserve-env=CRYPTOLIB,GITHUB_ACTIONS,CI .github/workflows/unit_tests.sh SETUP - - name: Build & test (${{ matrix.run_phase }}-${{ matrix.cryptolib }}) + - name: Build & test run: sudo --preserve-env=CRYPTOLIB,GITHUB_ACTIONS,CI .github/workflows/unit_tests.sh RUN_${{ matrix.run_phase }} env: CRYPTOLIB: ${{ matrix.cryptolib }} From 592ee08f3b8112e5fc3c2881d06365192c297b86 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 5 Sep 2023 11:29:06 +0200 Subject: [PATCH 2/5] ci: Use add-apt-repository to enable sources This should also ensure that consistent mirrors are selected. --- .github/workflows/build_test.sh | 7 +++---- .github/workflows/unit_tests.sh | 5 ++--- test/README.testsuite | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_test.sh b/.github/workflows/build_test.sh index a512e5101d2..aa4a25d7c5f 100755 --- a/.github/workflows/build_test.sh +++ b/.github/workflows/build_test.sh @@ -68,8 +68,6 @@ LINKER="${LINKER:?}" CRYPTOLIB="${CRYPTOLIB:?}" RELEASE="$(lsb_release -cs)" -bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ $RELEASE main restricted universe multiverse' >>/etc/apt/sources.list" - # Note: As we use postfixed clang/gcc binaries, we need to override $AR # as well, otherwise meson falls back to ar from binutils which # doesn't work with LTO @@ -96,7 +94,7 @@ elif [[ "$COMPILER" == gcc ]]; then if ! apt install --dry-run "gcc-$COMPILER_VERSION" >/dev/null; then # Latest gcc stack deb packages provided by # https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test - add-apt-repository -y ppa:ubuntu-toolchain-r/test + add-apt-repository -y --no-update ppa:ubuntu-toolchain-r/test fi PACKAGES+=("gcc-$COMPILER_VERSION" "gcc-$COMPILER_VERSION-multilib") @@ -105,7 +103,8 @@ else fi # PPA with some newer build dependencies (like zstd) -add-apt-repository -y ppa:upstream-systemd-ci/systemd-ci +add-apt-repository -y --no-update ppa:upstream-systemd-ci/systemd-ci +add-apt-repository -y --no-update --enable-source apt-get -y update apt-get -y build-dep systemd apt-get -y install "${PACKAGES[@]}" diff --git a/.github/workflows/unit_tests.sh b/.github/workflows/unit_tests.sh index 0985817b72e..a5b98e089be 100755 --- a/.github/workflows/unit_tests.sh +++ b/.github/workflows/unit_tests.sh @@ -3,7 +3,6 @@ # shellcheck disable=SC2206 PHASES=(${@:-SETUP RUN RUN_ASAN_UBSAN CLEANUP}) -RELEASE="$(lsb_release -cs)" ADDITIONAL_DEPS=( clang expect @@ -46,9 +45,9 @@ for phase in "${PHASES[@]}"; do case $phase in SETUP) info "Setup phase" - bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ $RELEASE main restricted universe multiverse' >>/etc/apt/sources.list" # PPA with some newer build dependencies - add-apt-repository -y ppa:upstream-systemd-ci/systemd-ci + add-apt-repository -y --no-update ppa:upstream-systemd-ci/systemd-ci + add-apt-repository -y --no-update --enable-source apt-get -y update apt-get -y build-dep systemd apt-get -y install "${ADDITIONAL_DEPS[@]}" diff --git a/test/README.testsuite b/test/README.testsuite index c0f1a2b102f..bd72f413d05 100644 --- a/test/README.testsuite +++ b/test/README.testsuite @@ -191,7 +191,7 @@ the PR (set by the $UPSTREAM_PULL_REQUEST env variable) you'd like to debug: Now install necessary build & test dependencies: ## PPA with some newer Ubuntu packages required by upstream systemd -# add-apt-repository -y ppa:upstream-systemd-ci/systemd-ci +# add-apt-repository -y --enable-source ppa:upstream-systemd-ci/systemd-ci # apt build-dep -y systemd # apt install -y autopkgtest debhelper genisoimage git qemu-system-x86 From 051ec23ce2dd0537adc98b9871995128e0b89fd7 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 5 Sep 2023 12:02:05 +0200 Subject: [PATCH 3/5] ci: Use apt-get in favor of apt Apparently, apt does not have a stable CLI interface and warns about it. --- .github/workflows/build_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test.sh b/.github/workflows/build_test.sh index aa4a25d7c5f..e01eb52ffe2 100755 --- a/.github/workflows/build_test.sh +++ b/.github/workflows/build_test.sh @@ -77,7 +77,7 @@ if [[ "$COMPILER" == clang ]]; then AR="llvm-ar-$COMPILER_VERSION" # Prefer the distro version if available - if ! apt install --dry-run "llvm-$COMPILER_VERSION" >/dev/null; then + if ! apt-get -y install --dry-run "llvm-$COMPILER_VERSION" >/dev/null; then # Latest LLVM stack deb packages provided by https://apt.llvm.org/ # Following snippet was partly borrowed from https://apt.llvm.org/llvm.sh wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --yes --dearmor --output /usr/share/keyrings/apt-llvm-org.gpg @@ -91,7 +91,7 @@ elif [[ "$COMPILER" == gcc ]]; then CXX="g++-$COMPILER_VERSION" AR="gcc-ar-$COMPILER_VERSION" - if ! apt install --dry-run "gcc-$COMPILER_VERSION" >/dev/null; then + if ! apt-get -y install --dry-run "gcc-$COMPILER_VERSION" >/dev/null; then # Latest gcc stack deb packages provided by # https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test add-apt-repository -y --no-update ppa:ubuntu-toolchain-r/test From ce2c01789c150090124abc73e638fda61deaff97 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 5 Sep 2023 12:03:13 +0200 Subject: [PATCH 4/5] ci: Don't produce debug output for build tests These binaries are never used, so generating debug symbols just slows down build time. --- .github/workflows/build_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test.sh b/.github/workflows/build_test.sh index e01eb52ffe2..837baba57e9 100755 --- a/.github/workflows/build_test.sh +++ b/.github/workflows/build_test.sh @@ -138,7 +138,7 @@ for args in "${ARGS[@]}"; do CXX="$CXX" CXX_LD="$LD" CXXFLAGS="-Werror" \ meson setup \ -Dtests=unsafe -Dslow-tests=true -Dfuzz-tests=true --werror \ - -Dnobody-group=nogroup -Dcryptolib="${CRYPTOLIB:?}" \ + -Dnobody-group=nogroup -Dcryptolib="${CRYPTOLIB:?}" -Ddebug=false \ $args build; then cat build/meson-logs/meson-log.txt From 690db0c80f9d2d085d6c2117be004b5b044ca5f7 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 5 Sep 2023 12:07:01 +0200 Subject: [PATCH 5/5] ci: Do not run build test as root Although, this is CI, we can still do better. It also ensures that any env var changes make it into the script, as things like PATH would not survive a `sudo -E`. --- .github/workflows/build_test.sh | 19 ++++++++++--------- .github/workflows/build_test.yml | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_test.sh b/.github/workflows/build_test.sh index 837baba57e9..c0872117abc 100755 --- a/.github/workflows/build_test.sh +++ b/.github/workflows/build_test.sh @@ -80,9 +80,10 @@ if [[ "$COMPILER" == clang ]]; then if ! apt-get -y install --dry-run "llvm-$COMPILER_VERSION" >/dev/null; then # Latest LLVM stack deb packages provided by https://apt.llvm.org/ # Following snippet was partly borrowed from https://apt.llvm.org/llvm.sh - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --yes --dearmor --output /usr/share/keyrings/apt-llvm-org.gpg - printf "deb [signed-by=/usr/share/keyrings/apt-llvm-org.gpg] http://apt.llvm.org/%s/ llvm-toolchain-%s-%s main\n" \ - "$RELEASE" "$RELEASE" "$COMPILER_VERSION" >/etc/apt/sources.list.d/llvm-toolchain.list + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \ + sudo gpg --yes --dearmor --output /usr/share/keyrings/apt-llvm-org.gpg + echo "deb [signed-by=/usr/share/keyrings/apt-llvm-org.gpg] http://apt.llvm.org/$RELEASE/ llvm-toolchain-$RELEASE-$COMPILER_VERSION main" | \ + sudo tee /etc/apt/sources.list.d/llvm-toolchain.list fi PACKAGES+=("clang-$COMPILER_VERSION" "lldb-$COMPILER_VERSION" "python3-lldb-$COMPILER_VERSION" "lld-$COMPILER_VERSION" "clangd-$COMPILER_VERSION") @@ -94,7 +95,7 @@ elif [[ "$COMPILER" == gcc ]]; then if ! apt-get -y install --dry-run "gcc-$COMPILER_VERSION" >/dev/null; then # Latest gcc stack deb packages provided by # https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test - add-apt-repository -y --no-update ppa:ubuntu-toolchain-r/test + sudo add-apt-repository -y --no-update ppa:ubuntu-toolchain-r/test fi PACKAGES+=("gcc-$COMPILER_VERSION" "gcc-$COMPILER_VERSION-multilib") @@ -103,11 +104,11 @@ else fi # PPA with some newer build dependencies (like zstd) -add-apt-repository -y --no-update ppa:upstream-systemd-ci/systemd-ci -add-apt-repository -y --no-update --enable-source -apt-get -y update -apt-get -y build-dep systemd -apt-get -y install "${PACKAGES[@]}" +sudo add-apt-repository -y --no-update ppa:upstream-systemd-ci/systemd-ci +sudo add-apt-repository -y --no-update --enable-source +sudo apt-get -y update +sudo apt-get -y build-dep systemd +sudo apt-get -y install "${PACKAGES[@]}" # Install more or less recent meson and ninja with pip, since the distro versions don't # always support all the features we need (like --optimization=). Since the build-dep # command above installs the distro versions, let's install the pip ones just diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index f8b0ccaf81d..ccbbe26ecdc 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -35,4 +35,4 @@ jobs: - name: Repository checkout uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 - name: Build check - run: sudo -E .github/workflows/build_test.sh + run: .github/workflows/build_test.sh