From 9c411859bc8d95ca965d66ca38bb69d36bab4534 Mon Sep 17 00:00:00 2001 From: bherbruck Date: Sun, 18 Oct 2026 20:35:03 -0400 Subject: [PATCH] CI: self-hosted test shards build in their warm directories, no 3 GB download (#2833) On main and in the merge queue every test shard ran on the self-hosted machine but waited for the GitHub-hosted Build tests job and then spent about 12.5 minutes downloading its 3.2 GB archive (four shards at once over one connection). There, each shard now builds the tests in the runner's warm build directory (the existing build-tests one) and runs its slice directly; Build tests runs only where the shards run on GitHub's or the untrusted runners. Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 54 +++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 102743b64..b8bb2e0b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,10 +23,12 @@ # build directories via scripts/ci/warm-target.sh). Fork PRs' Linux jobs run on the isolated, ephemeral # `self-hosted-untrusted` runners (no warm caches) when the PR number ends in 7, 8 or 9 (the rest of # the fork PRs, and everything else, run on GitHub's runners, which have far more free slots). -# The test archive is always built on GitHub's runners: the self-hosted machine has a newer -# glibc, so its binaries wouldn't start on ubuntu-24.04, while the archive runs there unchanged -# (same /home/runner/work checkout path). Fork pull requests, the nightly run and Run workflow -# use GitHub's runners only; the self-hosted runners refuse any other job. +# Where the test shards run on GitHub's runners, the test archive is built there too: the +# self-hosted machine has a newer glibc, so its binaries wouldn't start on ubuntu-24.04, while the +# archive runs there unchanged (same /home/runner/work checkout path). The self-hosted shards +# (main and the merge queue) skip the archive and build the tests in their warm build directories. +# Fork pull requests, the nightly run and Run workflow use GitHub's runners only; the self-hosted +# runners refuse any other job. name: CI on: @@ -270,7 +272,9 @@ jobs: build-tests: name: Build tests needs: changes - if: needs.changes.outputs.native == 'true' + # The self-hosted shards (main and the merge queue) build from their warm build directories + # instead: downloading the 3 GB archive took them longer than building. + if: needs.changes.outputs.native == 'true' && needs.changes.outputs.local != 'true' runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04' }} # Fork builds compile inside gVisor on shared boxes; give them room to finish and upload. timeout-minutes: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 90 || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 60 || 45 }} @@ -317,6 +321,7 @@ jobs: tests: name: Native Linux tests (${{ matrix.shard }}/4) needs: [changes, build-tests] + if: ${{ !cancelled() && needs.changes.outputs.native == 'true' && (needs.changes.outputs.local == 'true' || needs.build-tests.result == 'success') }} runs-on: ${{ needs.changes.outputs.local == 'true' && 'self-hosted-fast' || ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (endsWith(format('{0}', github.event.pull_request.number), '7') || endsWith(format('{0}', github.event.pull_request.number), '8') || endsWith(format('{0}', github.event.pull_request.number), '9'))) && 'self-hosted-untrusted' || 'ubuntu-24.04') }} timeout-minutes: 60 strategy: @@ -341,12 +346,26 @@ jobs: - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 with: tool: cargo-nextest - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + # On the self-hosted runners each shard builds the tests in its warm build directory (a few + # minutes for what changed); elsewhere it runs the archive the Build tests job made. + - name: Size parallel compiles to the job's memory + if: needs.changes.outputs.local == 'true' + run: scripts/ci/build-jobs.sh + - name: Warm build directory (self-hosted runner) + if: needs.changes.outputs.local == 'true' + run: scripts/ci/warm-target.sh build-tests + - if: needs.changes.outputs.local == 'true' + uses: dtolnay/rust-toolchain@stable + - if: needs.changes.outputs.local != 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: nextest-archive - name: Run tests env: EVENT_NAME: ${{ github.event_name }} + LOCAL: ${{ needs.changes.outputs.local }} + FULL: ${{ needs.changes.outputs.full }} + PACKAGES: ${{ needs.changes.outputs.packages }} SHARD: ${{ matrix.shard }} SHARDS: ${{ github.event_name == 'pull_request' && 2 || 4 }} run: | @@ -357,10 +376,19 @@ jobs: if [ "$EVENT_NAME" != pull_request ]; then all=(--ignore-default-filter -E 'not (package(solidcraft-kernel) & binary(boolean_props))') fi - # Extract into ./target: tests that run a built binary (env!("CARGO_BIN_EXE_*")) have - # the build job's path, which is the same checkout path here. - cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \ - --profile ci --partition "slice:$SHARD/$SHARDS" --no-tests=pass "${all[@]}" + if [ "$LOCAL" = true ]; then + scope=(--workspace) + if [ "$FULL" != true ]; then scope=(); for p in $PACKAGES; do scope+=(-p "$p"); done; fi + # The warm build directory keeps the last run's junit.xml: never upload that one. + rm -f "$CARGO_TARGET_DIR/nextest/ci/junit.xml" + cargo nextest run "${scope[@]}" --cargo-profile ci --locked \ + --profile ci --partition "slice:$SHARD/$SHARDS" --no-tests=pass "${all[@]}" + else + # Extract into ./target: tests that run a built binary (env!("CARGO_BIN_EXE_*")) have + # the build job's path, which is the same checkout path here. + cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \ + --profile ci --partition "slice:$SHARD/$SHARDS" --no-tests=pass "${all[@]}" + fi - name: Robustness goldens (pull requests that change geometry) # Every seed of tests/robustness/goldens.json (each seed is a test), split over the shards; # pushes to main and the nightly run them in the step above. @@ -381,7 +409,7 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: junit-${{ matrix.shard }} - path: target/nextest/ci/junit.xml + path: ${{ env.CARGO_TARGET_DIR || 'target' }}/nextest/ci/junit.xml retention-days: 7 action-matrix: @@ -638,6 +666,7 @@ jobs: CHANGES: ${{ needs.changes.result }} CHECKS: ${{ needs.checks.result }} NATIVE: ${{ needs.changes.outputs.native }} + LOCAL: ${{ needs.changes.outputs.local }} WINDOWS: ${{ needs.changes.outputs.windows }} ORACLE_RUN: ${{ needs.changes.outputs.oracle }} CLIPPY: ${{ needs.clippy.result }} @@ -652,7 +681,8 @@ jobs: test "$CHECKS" = success if [ "$NATIVE" = true ]; then test "$CLIPPY" = success - test "$BUILD" = success + # The self-hosted test shards build their own tests (no Build tests job there). + [ "$BUILD" = success ] || { [ "$LOCAL" = true ] && [ "$BUILD" = skipped ]; } test "$TESTS" = success test "$DOCS" = success fi