From 7d222741581e0589010c49d14edd174ca6521c5b Mon Sep 17 00:00:00 2001 From: Justin Scroggins Date: Wed, 11 Nov 2026 12:15:27 -0500 Subject: [PATCH] Fail CI in minutes, not an hour, when Ubuntu's mirror stops answering On 2026-11-11 the GitHub runners could not reach Ubuntu's package mirror. apt waited out every package in turn, and the Linux jobs sat in setup for 48 to 58 minutes before anything built. Nothing bounded them: no job had a timeout, so GitHub's six hours was the limit. apt.sh now fails the update when an index does not download, gives each request 20 seconds, caps the update at 2 minutes and the download at 10, and tries once more after a minute. The install reads only what was downloaded and is never cut short. Against a mirror that does not answer it fails in under two minutes; against a working one it installs as before. Every CI, Mobile and Release build job also has a timeout, about twice its slowest run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 ++ .github/workflows/mobile.yml | 2 ++ .github/workflows/release.yml | 4 ++++ packaging/ci/apt.sh | 38 ++++++++++++++++++++++++++++++----- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0562df7a..fa368c52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: - os: macos-latest features: --features tinyclient/gui runs-on: ${{ matrix.os }} + timeout-minutes: 60 steps: - uses: actions/checkout@v5 - uses: ./.github/actions/setup-rust @@ -79,6 +80,7 @@ jobs: artifact: ${{ fromJSON(github.event_name == 'push' && '["linux-x86_64","linux-aarch64","macos","windows"]' || '["linux-x86_64"]') }} # Ubuntu 22.04 is the oldest glibc the release packages promise. runs-on: ${{ fromJSON('{"linux-x86_64":"ubuntu-22.04","linux-aarch64":"ubuntu-22.04-arm","macos":"macos-latest","windows":"windows-latest"}')[matrix.artifact] }} + timeout-minutes: 45 steps: - uses: actions/checkout@v5 # Linux aarch64 builds without a cache. It runs only on main, beside the longer macOS diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index f6e6f313..64c423f2 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -63,6 +63,7 @@ jobs: android-app: name: android (libs + assembleDebug) runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v5 @@ -168,6 +169,7 @@ jobs: # Xcode is macOS-only. macos-26 carries Xcode 26.6 / iOS 26.5 SDK, which satisfies the # current mandate; move to the xcode-27 preview image before April 2027. runs-on: macos-26 + timeout-minutes: 30 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 432912af..7663bbc1 100755 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,6 +105,7 @@ jobs: macos: needs: wait-for-ci runs-on: macos-latest + timeout-minutes: 90 steps: - uses: actions/checkout@v5 - name: Name the files @@ -169,6 +170,7 @@ jobs: runner: ubuntu-22.04-arm artifact: linux-aarch64 runs-on: ${{ matrix.runner }} + timeout-minutes: 90 steps: - uses: actions/checkout@v5 - name: Name the files @@ -246,6 +248,7 @@ jobs: image: opensuse/tumbleweed kind: rpm runs-on: ${{ matrix.runner }} + timeout-minutes: 30 steps: - uses: actions/checkout@v5 - name: Name the files @@ -260,6 +263,7 @@ jobs: windows: needs: wait-for-ci runs-on: windows-latest + timeout-minutes: 90 steps: - uses: actions/checkout@v5 - name: Name the files diff --git a/packaging/ci/apt.sh b/packaging/ci/apt.sh index da48f84a..5947ac52 100755 --- a/packaging/ci/apt.sh +++ b/packaging/ci/apt.sh @@ -2,10 +2,38 @@ # Libraries the Linux build and the packaging scripts link and scan. # mold is the linker. dpkg-dev is what linux.sh uses to write the .deb dependencies, and patchelf # what it points the AppImage's tinyclient at the libraries bundled beside it. +# +# A mirror that stops answering fails this in minutes instead of holding the job: when the +# runners could not resolve Ubuntu's mirror, apt waited out every package in turn and Linux +# jobs sat here for most of an hour. Each request gives up after 20 seconds, and an index that +# does not download fails the update instead of being ignored, so a dead mirror stops here. The +# update has 2 minutes and the download 10; the install itself reads only what was downloaded +# and is never cut short, because a dpkg stopped halfway breaks the next try. A failed try is +# tried once more after a minute, in case the mirror was only blinking. set -eu -sudo apt-get update -sudo apt-get install -y \ - build-essential pkg-config libssl-dev libasound2-dev libgtk-3-dev libwebkit2gtk-4.1-dev \ - libsoup-3.0-dev libvulkan-dev libwayland-dev libxkbcommon-dev libxkbcommon-x11-dev libx11-dev \ - libxcb1-dev libfontconfig1-dev librsvg2-bin desktop-file-utils dpkg-dev patchelf mold +packages="build-essential pkg-config libssl-dev libasound2-dev libgtk-3-dev libwebkit2gtk-4.1-dev + libsoup-3.0-dev libvulkan-dev libwayland-dev libxkbcommon-dev libxkbcommon-x11-dev libx11-dev + libxcb1-dev libfontconfig1-dev librsvg2-bin desktop-file-utils dpkg-dev patchelf mold" + +# apt +apt() { + limit=$1 + shift + sudo timeout "$limit" apt-get -o Acquire::Retries=2 -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 -o DPkg::Lock::Timeout=60 "$@" +} + +for try in 1 2; do + # shellcheck disable=SC2086 # the package list splits into words on purpose + if apt 120 update --error-on=any && apt 600 install -y --download-only $packages; then + # shellcheck disable=SC2086 + exec sudo apt-get install -y --no-download $packages + fi + if [ "$try" = 1 ]; then + echo "::warning::apt did not finish; trying once more in a minute" + sleep 60 + fi +done +echo "::error::Ubuntu's package mirror did not answer. This is the runner's network, not the build: re-run the job." +exit 1