diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d37b32a34..78ed5da7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,16 @@ jobs: # shared-key by profile + runner image (not job or package-manager # version). rust-cache also hashes the toolchain, Cargo manifests # and Rust environment. Check/coverage/release keep separate caches. + # A cache restore is an optimisation, so cap it and carry on cold. + # Healthy restores take under 30 s on every runner OS. On 2026-10-26 + # one hung for 14+ min in yarn-classic-matrix and held merge-queue + # head #1719 (and 12 entries behind it) until it was cancelled; + # uncapped it runs to the job's own timeout. Every cache restore in + # this file carries the same two keys (enforced by + # scripts/tests/test_merge_queue_fail_fast.py). uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: save-if: ${{ github.ref == 'refs/heads/main' }} @@ -150,6 +159,8 @@ jobs: - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -393,6 +404,8 @@ jobs: # Swatinem/rust-cache, main-only saves: see the first `Cache cargo` # step in this file. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-${{ matrix.os }} save-if: ${{ github.ref == 'refs/heads/main' }} @@ -431,6 +444,9 @@ jobs: id: vexctl-cache if: needs.release-readiness.outputs.reuse != 'true' && runner.os == 'macOS' uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + # Capped like `Cache cargo`: a miss only means compiling vexctl. + timeout-minutes: 5 + continue-on-error: true with: path: ${{ runner.temp }}/vexctl-bin key: vexctl-${{ env.VEXCTL_VERSION }}-go${{ steps.go.outputs.go-version }}-${{ runner.os }}-${{ runner.arch }} @@ -504,6 +520,7 @@ jobs: && runner.os == 'macOS' && github.ref == 'refs/heads/main' && steps.vexctl-cache.outputs.cache-hit != 'true' + && steps.vexctl-cache.outcome == 'success' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ runner.temp }}/vexctl-bin @@ -569,6 +586,8 @@ jobs: # Swatinem/rust-cache, main-only saves: see the first `Cache cargo` # step in this file. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: save-if: ${{ github.ref == 'refs/heads/main' }} @@ -633,6 +652,8 @@ jobs: # The shards share one cache key and shard 1 alone writes it. # rust-cache keeps only dependencies, which every shard builds. uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: save-if: ${{ github.ref == 'refs/heads/main' && matrix.shard == 1 }} @@ -1055,6 +1076,8 @@ jobs: - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: e2e-${{ matrix.os }} save-if: ${{ github.ref == 'refs/heads/main' }} @@ -1121,6 +1144,8 @@ jobs: - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: check-windows-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2281,6 +2306,8 @@ jobs: run: scripts/rustup-retry.sh toolchain install - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2326,6 +2353,8 @@ jobs: run: scripts/rustup-retry.sh toolchain install - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-${{ matrix.os }} save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2561,6 +2590,8 @@ jobs: docker pull rust:1.56-slim - name: Cache cargo uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} @@ -2598,7 +2629,8 @@ jobs: # The kill switch gates the STEPS, never the job. # * NO `needs:` — an upstream failure would skip this job, same wedge. # * NO matrix and NO rename — the check name must stay `hosted-e2e`. - # * NO `continue-on-error` — a bypass must be visible, not invisible. + # * NO `continue-on-error` on the job or its e2e steps — a bypass must be + # visible, not invisible. Only the capped `Cache cargo` restore has it. # The job ALWAYS runs and ALWAYS reaches success or failure. # # ESCAPE HATCH — when production is down and this is blocking merges: @@ -2669,6 +2701,8 @@ jobs: - name: Cache cargo if: steps.gate.outputs.run == 'true' uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + timeout-minutes: 5 + continue-on-error: true with: shared-key: dev-ubuntu-latest save-if: ${{ github.ref == 'refs/heads/main' }} diff --git a/scripts/merge-queue-fail-fast.py b/scripts/merge-queue-fail-fast.py index 452208d22..b3d6be1cf 100644 --- a/scripts/merge-queue-fail-fast.py +++ b/scripts/merge-queue-fail-fast.py @@ -7,7 +7,7 @@ ~45 minutes the Gradle e2e legs take, burning macOS and Windows runners on a run that could no longer pass. -No CI job sets `continue-on-error`, so one failed or timed-out job means +No CI job sets a job-level `continue-on-error`, so one failed or timed-out job means `ci-ok` will fail. Cancelling the run then changes only *when* it fails: `ci-ok` is `if: always()`, so it still runs on the cancelled run, sees the failed and cancelled `needs`, and reports failure within seconds, and the diff --git a/scripts/tests/test_merge_queue_fail_fast.py b/scripts/tests/test_merge_queue_fail_fast.py index 7efefb9f3..1b7a6f64b 100644 --- a/scripts/tests/test_merge_queue_fail_fast.py +++ b/scripts/tests/test_merge_queue_fail_fast.py @@ -115,7 +115,24 @@ class CiWorkflowContract(unittest.TestCase): def test_no_job_tolerates_failure(self): # A job-level continue-on-error would let ci-ok pass despite a failed # job, and this script would then cancel a run that could still land. - self.assertIsNone(re.search(r"^\s*continue-on-error:", self.text, re.M)) + # A step-level one hides that step's failure the same way, so only + # the capped cache restores may carry it: a miss there just means + # building cold. + self.assertIsNone(re.search(r"^ continue-on-error:", self.text, re.M)) + steps = re.split(r"(?m)^ - ", self.text) + tolerant = [s for s in steps if re.search(r"(?m)^\s*continue-on-error:", s)] + self.assertTrue(tolerant) + for step in tolerant: + self.assertRegex(step, r"(?m)^ uses: (Swatinem/rust-cache|actions/cache/restore)@", step[:80]) + self.assertRegex(step, r"(?m)^ timeout-minutes: \d+$", step[:80]) + self.assertRegex(step, r"(?m)^ continue-on-error: true$", step[:80]) + + def test_every_cache_restore_is_capped(self): + # An uncapped restore that hangs holds the queue head until the job + # times out (#1730). + for step in re.split(r"(?m)^ - ", self.text): + if re.search(r"(?m)^ uses: (Swatinem/rust-cache|actions/cache/restore)@", step): + self.assertRegex(step, r"(?m)^ timeout-minutes: \d+$", step[:80]) def test_gate_runs_on_a_cancelled_run(self): gate = self.text[self.text.index("\n ci-ok:"):]