From a363346949a774346bbe2d50aa3294231d740bf8 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:52:29 -0500 Subject: [PATCH] refactor(ci): share bounded hosted package acquisition (#12407) --- .github/actions/host-apt-packages/action.yml | 37 ++++++++++++ .../actions/windows-cross-compiler/action.yml | 18 ++---- .github/workflows/ci.yml | 2 +- .github/workflows/real-binary-layout.yml | 8 ++- .github/workflows/real-signal-recording.yml | 4 +- scripts/ci/scopes.mjs | 3 + .../boundary/filesystem/ciChangeScope.test.ts | 11 ++++ .../filesystem/ciRequiredLanes.test.ts | 60 +++++++++++++++++++ 8 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 .github/actions/host-apt-packages/action.yml diff --git a/.github/actions/host-apt-packages/action.yml b/.github/actions/host-apt-packages/action.yml new file mode 100644 index 0000000000..c07d7d3989 --- /dev/null +++ b/.github/actions/host-apt-packages/action.yml @@ -0,0 +1,37 @@ +name: Install hosted Ubuntu packages +description: Download fixture prerequisites within a deadline, then install from the local APT cache. +inputs: + packages: + description: Space-separated package names from the already selected signed sources. + required: true + download-timeout-seconds: + description: Outer download deadline in seconds, before the five-second termination grace. + default: "180" + install-recommends: + description: Leave APT's recommendation policy in effect when true. + default: "true" +runs: + using: composite + steps: + - shell: bash + env: + REA_APT_PACKAGES: ${{ inputs.packages }} + REA_APT_DOWNLOAD_TIMEOUT: ${{ inputs.download-timeout-seconds }} + REA_APT_INSTALL_RECOMMENDS: ${{ inputs.install-recommends }} + run: | + read -r -a packages <<< "$REA_APT_PACKAGES" + if (( ${#packages[@]} == 0 )); then + echo '::error::No hosted Ubuntu packages were supplied' + exit 1 + fi + options=(--yes) + if [[ "$REA_APT_INSTALL_RECOMMENDS" == 'false' ]]; then + options+=(--no-install-recommends) + fi + sudo timeout --kill-after=5s "${REA_APT_DOWNLOAD_TIMEOUT}s" \ + apt-get -o Acquire::Retries=2 --download-only install "${options[@]}" -- "${packages[@]}" || { + status=$? + echo "::error::APT package download failed (exit ${status})" + exit "$status" + } + sudo apt-get --no-download install "${options[@]}" -- "${packages[@]}" diff --git a/.github/actions/windows-cross-compiler/action.yml b/.github/actions/windows-cross-compiler/action.yml index 51878faa24..433d21298c 100644 --- a/.github/actions/windows-cross-compiler/action.yml +++ b/.github/actions/windows-cross-compiler/action.yml @@ -4,19 +4,13 @@ runs: using: composite steps: - uses: ./.github/actions/ubuntu-archive + # Install and select only the POSIX variant used by the Windows add-on. + - uses: ./.github/actions/host-apt-packages + with: + packages: g++-mingw-w64-x86-64-posix + download-timeout-seconds: "120" + install-recommends: "false" - shell: bash run: | - # The addon uses std::mutex. Install one threading model and select - # it explicitly so preinstalled alternatives cannot choose another. - compiler=(--no-install-recommends -y g++-mingw-w64-x86-64-posix) - # Bound downloads separately from dpkg. Together with the index - # refresh this allows at most 290 seconds for archive acquisition. - sudo timeout --kill-after=5s 120s \ - apt-get -o Acquire::Retries=2 --download-only install "${compiler[@]}" || { - status=$? - echo "MinGW package download failed (exit ${status})" >&2 - exit "$status" - } - sudo apt-get --no-download install "${compiler[@]}" sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix x86_64-w64-mingw32-g++ -v diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1afb9057c..bc282db943 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -699,7 +699,7 @@ jobs: - name: Compute Windows native cache key id: windows-native-key env: - SOURCE_HASH: ${{ hashFiles('native/windows/**', 'src/domain/windowsNativeArtifact*.ts', 'src/domain/dosMz*.ts', 'src/domain/peCoffHeader*.ts', 'scripts/build-windows-native.mjs', 'scripts/verify-windows-native-artifact.mjs', 'scripts/lib/verifier-source-loader.mjs', 'tests/fixtures/windows-native-inspection-failure.cc', 'tests/fixtures/windows/processBoundary.cc', '.github/actions/windows-cross-compiler/action.yml', '.github/actions/ubuntu-archive/action.yml', 'package.json') }} + SOURCE_HASH: ${{ hashFiles('native/windows/**', 'src/domain/windowsNativeArtifact*.ts', 'src/domain/dosMz*.ts', 'src/domain/peCoffHeader*.ts', 'scripts/build-windows-native.mjs', 'scripts/verify-windows-native-artifact.mjs', 'scripts/lib/verifier-source-loader.mjs', 'tests/fixtures/windows-native-inspection-failure.cc', 'tests/fixtures/windows/processBoundary.cc', '.github/actions/windows-cross-compiler/action.yml', '.github/actions/ubuntu-archive/action.yml', '.github/actions/host-apt-packages/action.yml', 'package.json') }} run: | utc_date="$(date -u +%Y-%m-%d)" printf 'key=rea-windows-native-v3-%s-%s-%s-%s-%s-%s\n' \ diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index ffa1633de1..b5ec508311 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -59,10 +59,12 @@ jobs: with: noble-azure-fallback: "true" - name: Install AArch64 and ELF32 cross toolchains for source-owned fixtures - run: | - sudo apt-get install --yes --no-install-recommends \ - gcc-aarch64-linux-gnu gcc-i686-linux-gnu gcc-arm-linux-gnueabi gcc-mips-linux-gnu gcc-mipsel-linux-gnu \ + uses: ./.github/actions/host-apt-packages + with: + packages: >- + gcc-aarch64-linux-gnu gcc-i686-linux-gnu gcc-arm-linux-gnueabi gcc-mips-linux-gnu gcc-mipsel-linux-gnu libc6-dev-arm64-cross libc6-dev-i386-cross libc6-dev-armel-cross libc6-dev-mips-cross libc6-dev-mipsel-cross + install-recommends: "false" - name: Acquire exact upstream libraries in an isolated runner environment shell: bash run: | diff --git a/.github/workflows/real-signal-recording.yml b/.github/workflows/real-signal-recording.yml index b9013eff34..8ca0b61e73 100644 --- a/.github/workflows/real-signal-recording.yml +++ b/.github/workflows/real-signal-recording.yml @@ -44,7 +44,9 @@ jobs: uses: ./.github/actions/npm-cache - uses: ./.github/actions/ubuntu-archive - name: Install signal recording prerequisites - run: sudo apt-get install -y sigrok-cli util-linux python3 + uses: ./.github/actions/host-apt-packages + with: + packages: sigrok-cli util-linux python3 - run: npm ci - uses: ./.github/actions/portable-build-cache with: diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 26f3d8ac82..1871017573 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -69,6 +69,7 @@ const PATH_SCOPES = { "scripts/verify-signal-recording.mjs", ".github/workflows/real-signal-recording.yml", ".github/actions/ubuntu-archive/action.yml", + ".github/actions/host-apt-packages/action.yml", ], readiness: [ "tests/conformance/readiness/**", @@ -250,6 +251,7 @@ const PATH_SCOPES = { "scripts/verify-windows-*.mjs", ".github/actions/windows-cross-compiler/action.yml", ".github/actions/ubuntu-archive/action.yml", + ".github/actions/host-apt-packages/action.yml", "scripts/build-windows-native.mjs", "scripts/windows-capabilities.mjs", "scripts/prepack.mjs", @@ -440,6 +442,7 @@ const PATH_SCOPES = { binary_layout: [ ".github/workflows/real-binary-layout.yml", ".github/actions/ubuntu-archive/action.yml", + ".github/actions/host-apt-packages/action.yml", "bridge/pwntools/**", "src/native/pwntools/**", "src/config/pwntoolsInputLimit*.ts", diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 216add16a9..99b6c9a548 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -1246,6 +1246,17 @@ it.each([ ".github/actions/ubuntu-archive/action.yml", { code: true, runtime: true, package: true, binary_layout: true }, ], + [ + ".github/actions/host-apt-packages/action.yml", + { + code: true, + runtime: true, + package: true, + binary_layout: true, + signal_recording: true, + recorded_crash: false, + }, + ], ["src/domain/peCoffHeader.ts", { code: true, runtime: true, package: true }], ["src/domain/dosMz.ts", { code: true, runtime: true, package: true }], [ diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index e96ae6ecb6..fd369b6e06 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -1159,3 +1159,63 @@ it("requires every client schema engine in the build job of every code change", ); expect(check?.run).toContain("$SCHEMA_CHECK_OPTIONS"); }); + +it.skipIf(process.platform === "win32").each([ + ["true", "180", "0"], + ["false", "120", "0"], + ["false", "120", "124"], +] as const)( + "downloads hosted packages within the selected deadline and keeps cached install outside it (%s, %s, %s)", + async (recommends, seconds, status) => { + const { run } = z + .object({ + runs: z.object({ steps: z.tuple([z.object({ run: z.string() })]) }), + }) + .parse( + parse( + await readFile( + new URL( + "../../../.github/actions/host-apt-packages/action.yml", + import.meta.url, + ), + "utf8", + ), + ), + ).runs.steps[0]; + const result = await executeWorkflowFixture("bash", [ + "-e", + "-c", + ` + REA_APT_INSTALL_RECOMMENDS=$1 + REA_APT_DOWNLOAD_TIMEOUT=$2 + REA_APT_PACKAGES="package-one package-two" + fixture_status=$3 + sudo() { + printf '%s\n' "$*" + if [[ $1 == timeout ]]; then return "$fixture_status"; fi + } + ` + + "\n" + + run, + "fixture", + recommends, + seconds, + status, + ]).then( + ({ stdout }) => ({ code: 0, stdout }), + (cause: unknown) => + z.object({ code: z.number(), stdout: z.string() }).parse(cause), + ); + const install = `install --yes${recommends === "false" ? " --no-install-recommends" : ""} -- package-one package-two`; + const commands = result.stdout + .trim() + .split("\n") + .filter((line) => !line.startsWith("::error::")); + expect(result.code).toBe(Number(status)); + expect(commands).toEqual([ + `timeout --kill-after=5s ${seconds}s apt-get -o Acquire::Retries=2 --download-only ${install}`, + ...(status === "0" ? [`apt-get --no-download ${install}`] : []), + ]); + if (status !== "0") expect(result.stdout).toContain(`(exit ${status})`); + }, +);