From 31901b8be5484c17820bcd8f58abb941cad2e807 Mon Sep 17 00:00:00 2001 From: Stella Huang <100439259+StellaHuang95@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:25:51 -0800 Subject: [PATCH 1/2] ci: repair runner setup and exact-base comparisons Restore explicit Linux Conda provisioning and supported native macOS runners. Measure exact base/head revisions with isolated coverage fixtures and shared-runner performance fixtures, validating source provenance, measurement context, and execution outcomes before the unchanged quality gates. Preserve ARM64 coverage, subprocess proof, inventories, and all regression budgets. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 4 +- .github/workflows/coverage-baseline.yml | 10 +- .github/workflows/coverage-macos.yml | 2 +- .github/workflows/coverage.yml | 267 +++++++++++++++++--- .github/workflows/perf-baseline.yml | 31 ++- .github/workflows/perf-tests.yml | 311 +++++++++++++++++++----- .github/workflows/pr-check.yml | 35 ++- docs/QUALITY_SNAPSHOTS.md | 41 +++- scripts/tests/test_quality_workflows.py | 249 +++++++++++++++++++ 9 files changed, 829 insertions(+), 121 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5f426a7..90ff4970 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,9 +26,9 @@ jobs: # target: aarch64-unknown-linux-gnu # - os: ubuntu-latest # target: arm-unknown-linux-gnueabihf - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin - - os: macos-14 + - os: macos-15 target: aarch64-apple-darwin # - os: ubuntu-latest # target: x86_64-unknown-linux-gnu diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index a17f7225..c8c9dcc4 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -46,10 +46,14 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Linux) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Check Conda version run: conda info --all diff --git a/.github/workflows/coverage-macos.yml b/.github/workflows/coverage-macos.yml index c7586a5c..a36763d8 100644 --- a/.github/workflows/coverage-macos.yml +++ b/.github/workflows/coverage-macos.yml @@ -12,7 +12,7 @@ permissions: jobs: coverage: - runs-on: macos-14 + runs-on: macos-15 timeout-minutes: 35 steps: - name: Checkout diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 43d3a62f..c774efca 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,12 +14,14 @@ permissions: pull-requests: write jobs: - coverage: - name: Coverage + measurements: + name: Coverage measurement (${{ matrix.platform }}, ${{ matrix.revision }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, windows-latest] + revision: [base, head] include: - os: ubuntu-latest target: x86_64-unknown-linux-musl @@ -33,11 +35,14 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Post Coverage Started Comment - if: github.event.pull_request.head.repo.full_name == github.repository + if: >- + matrix.revision == 'head' && + github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: @@ -64,10 +69,14 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Linux) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Check Conda version run: conda info --all @@ -86,7 +95,12 @@ jobs: run: pipenv --version - name: Create a Pipenv Environment - run: pipenv install + id: pipenv + run: | + pipenv install + prefix="$(pipenv --venv)" + echo "prefix=$prefix" >> "$GITHUB_OUTPUT" + shell: bash - name: Install virtualenvwrapper (Linux) if: startsWith(matrix.os, 'ubuntu') @@ -197,6 +211,85 @@ jobs: run: cargo fetch shell: bash + - name: Capture Coverage Measurement Context + env: + MEASUREMENT_TARGET: ${{ matrix.target }} + CREATED_PIPENV_PREFIX: ${{ steps.pipenv.outputs.prefix }} + run: | + git rev-parse HEAD > source-revision.txt + python - <<'PY' + import json + import os + import shutil + import subprocess + from pathlib import Path + + def output(*arguments): + executable = shutil.which(arguments[0]) + if executable is None: + raise RuntimeError(f"Required measurement tool not found: {arguments[0]}") + value = subprocess.check_output( + [executable, *arguments[1:]], text=True, encoding="utf-8" + ).strip() + if not value: + raise RuntimeError(f"Empty measurement version: {arguments[0]}") + return value + + runner = {key: os.environ[key] for key in ( + "RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion" + )} + if not all(value.strip() for value in runner.values()): + raise RuntimeError("Runner image context must not be empty") + windows = os.name == "nt" + python_name = "python.exe" if windows else "python" + venv_bin = "Scripts" if windows else "bin" + pipenv_prefix = Path(os.environ["CREATED_PIPENV_PREFIX"]) + interpreters = { + name: output(str(prefix / venv_bin / python_name), "--version") + for name, prefix in ( + ("venv", Path(".venv")), + ("venv2", Path(".venv2")), + ("pipenv", pipenv_prefix), + ) + } + for name in ("default", "dev"): + prefix = Path(".pixi") / "envs" / name + executable = prefix / python_name if windows else prefix / "bin" / python_name + interpreters[f"pixi-{name}"] = output(str(executable), "--version") + for label, selector in ( + ("base", ("--name", "base")), + ("named", ("--name", "test-env1")), + ("prefix", ("--prefix", str(Path("prefix-envs/.conda1").resolve()))), + ): + interpreters[f"conda-{label}"] = output( + "conda", "run", *selector, "python", "--version" + ) + interpreters["pyenv"] = "\n".join(sorted( + output("pyenv", "versions", "--bare").splitlines() + )) + context = { + "runner": runner, + "installed_target": os.environ["MEASUREMENT_TARGET"], + "coverage_target": "host", + "features": "ci", + "versions": { + "rustc": output("rustc", "-vV"), + "cargo": output("cargo", "--version"), + "cargo-llvm-cov": output("cargo", "llvm-cov", "--version"), + "python": output("python", "--version"), + "conda": output("conda", "--version"), + "pipenv": output("pipenv", "--version"), + "pyenv": output("pyenv", "--version"), + "pixi": output("pixi", "--version"), + }, + "interpreters": interpreters, + } + Path("measurement-context.json").write_text( + json.dumps(context, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + PY + shell: bash + - name: Run Tests with Coverage id: coverage run: cargo llvm-cov --features ci --lcov --output-path lcov.info -- --nocapture --test-threads=1 @@ -220,42 +313,143 @@ jobs: --output production-coverage shell: bash - - name: Wait for Exact PR Base Coverage - id: wait_for_base_coverage + - name: Upload Coverage Measurement if: always() - run: >- - python scripts/wait_for_baseline.py - --repository "${{ github.repository }}" - --workflow coverage-baseline.yml - --commit "${{ github.event.pull_request.base.sha }}" - --artifact "coverage-baseline-${{ matrix.os }}" - --timeout-seconds 1200 - --poll-seconds 20 - env: - GITHUB_TOKEN: ${{ github.token }} - shell: bash + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-measurement-${{ matrix.os }}-${{ matrix.revision }}-${{ github.run_attempt }} + path: | + lcov.info + measurement-context.json + source-revision.txt + production-coverage/ + subprocess-coverage/ + subprocess-coverage.json + if-no-files-found: error + + coverage: + name: Coverage + needs: measurements + if: always() + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + platform: Linux + comment_header: coverage-linux + - os: windows-latest + target: x86_64-pc-windows-msvc + platform: Windows + comment_header: coverage-windows + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set Python to PATH + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Download Head Coverage Measurement + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: coverage-measurement-${{ matrix.os }}-head-${{ github.run_attempt }} - name: Download Exact PR Base Coverage - if: always() && steps.wait_for_base_coverage.outcome == 'success' - uses: dawidd6/action-download-artifact@eab87c9830c39eff17e5a6eadb20bfb4bc880477 # v27 + if: always() + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - workflow: coverage-baseline.yml - commit: ${{ github.event.pull_request.base.sha }} - workflow_conclusion: success - name: coverage-baseline-${{ matrix.os }} + name: coverage-measurement-${{ matrix.os }}-base-${{ github.run_attempt }} path: baseline-coverage - check_artifacts: true - search_artifacts: true - name: Compare Coverage Snapshot if: always() - run: >- - python scripts/quality_snapshot.py coverage - --current lcov.info - --baseline baseline-coverage/lcov.info - --platform "${{ matrix.platform }}" - --report coverage-report.md - --summary "$GITHUB_STEP_SUMMARY" + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.sha }} + MEASUREMENT_TARGET: ${{ matrix.target }} + MEASUREMENTS_RESULT: ${{ needs.measurements.result }} + PLATFORM: ${{ matrix.platform }} + run: | + python - <<'PY' + import argparse + import os + import re + import sys + from pathlib import Path + + sys.path.insert(0, "scripts") + from quality_snapshot import SnapshotError, load_json, run_coverage, write_report + + report = Path("coverage-report.md") + summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) + platform = os.environ["PLATFORM"] + + def context(path): + value = load_json(path) + required = { + "runner": ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"), + "versions": ( + "rustc", "cargo", "cargo-llvm-cov", "python", + "conda", "pipenv", "pyenv", "pixi", + ), + "interpreters": ( + "conda-base", "conda-named", "conda-prefix", "venv", "venv2", + "pipenv", "pixi-default", "pixi-dev", "pyenv", + ), + } + for group, keys in required.items(): + values = value.get(group) + if not isinstance(values, dict) or any( + not isinstance(values.get(key), str) or not values[key].strip() + for key in keys + ): + raise SnapshotError(f"Missing {group} measurement context in {path}") + if ( + value.get("features") != "ci" + or value.get("coverage_target") != "host" + or value.get("installed_target") != os.environ["MEASUREMENT_TARGET"] + ): + raise SnapshotError(f"Incompatible coverage workload in {path}") + return value + + try: + if os.environ.get("MEASUREMENTS_RESULT") != "success": + raise SnapshotError("One or more coverage measurements or required proofs failed") + for filename, variable in ( + ("source-revision.txt", "HEAD_SHA"), + ("baseline-coverage/source-revision.txt", "BASE_SHA"), + ): + expected = os.environ.get(variable, "") + if ( + re.fullmatch(r"[0-9a-f]{40}", expected) is None + or Path(filename).read_text(encoding="utf-8").strip() != expected + ): + raise SnapshotError(f"Unexpected source revision in {filename}") + current = context(Path("measurement-context.json")) + baseline = context(Path("baseline-coverage/measurement-context.json")) + if current != baseline: + raise SnapshotError( + "Coverage runner, tool, or fixture versions differ; inspect both measurement contexts" + ) + except (OSError, UnicodeError, SnapshotError) as error: + print(f"Invalid coverage measurement: {error}", file=sys.stderr) + write_report( + f"## Test Coverage Report ({platform})\n\n:x: **Invalid snapshot:** {error}\n", + report, summary, + ) + raise SystemExit(1) + + raise SystemExit(run_coverage(argparse.Namespace( + current=Path("lcov.info"), baseline=Path("baseline-coverage/lcov.info"), + platform=platform, report=report, summary=summary, + ))) + PY shell: bash - name: Upload PR Coverage Artifact @@ -265,6 +459,9 @@ jobs: name: coverage-pr-${{ matrix.os }} path: | lcov.info + measurement-context.json + source-revision.txt + baseline-coverage/ coverage-report.md production-coverage/ subprocess-coverage/ diff --git a/.github/workflows/perf-baseline.yml b/.github/workflows/perf-baseline.yml index 23821061..7830f80b 100644 --- a/.github/workflows/perf-baseline.yml +++ b/.github/workflows/perf-baseline.yml @@ -24,8 +24,9 @@ jobs: target: x86_64-unknown-linux-musl - os: windows-latest target: x86_64-pc-windows-msvc - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin + conda_arch: x86_64 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -45,20 +46,34 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Linux) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Install Conda + add to PATH (macOS) if: startsWith(matrix.os, 'macos') + env: + CONDA_ARCH: ${{ matrix.conda_arch }} run: | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - bash ~/miniconda.sh -b -p ~/miniconda - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV + if [[ "$(uname -m)" != "$CONDA_ARCH" ]]; then + echo "Conda installer architecture $CONDA_ARCH does not match runner $(uname -m)" >&2 + exit 1 + fi + curl --fail --location --show-error --output "$RUNNER_TEMP/miniconda.sh" \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh" + bash "$RUNNER_TEMP/miniconda.sh" -b -p "$HOME/miniconda" + echo "PATH=$PATH:$HOME/miniconda/bin" >> "$GITHUB_ENV" + echo "CONDA=$HOME/miniconda" >> "$GITHUB_ENV" shell: bash + - name: Check Conda version + run: conda info --all + - name: Create test Conda environment run: conda create -n perf-test-env python=3.12 -y diff --git a/.github/workflows/perf-tests.yml b/.github/workflows/perf-tests.yml index 0ccdf8d4..703c3868 100644 --- a/.github/workflows/perf-tests.yml +++ b/.github/workflows/perf-tests.yml @@ -30,13 +30,17 @@ jobs: target: x86_64-unknown-linux-musl platform: Linux comment_header: perf-linux - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin + conda_arch: x86_64 platform: macOS comment_header: perf-macos steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false - name: Post In-Progress Comment if: >- @@ -49,7 +53,7 @@ jobs: message: | ## Performance Report (${{ matrix.platform }}) :hourglass_flowing_sand: - Running performance tests against baseline `${{ github.event.pull_request.base.sha }}`. + Measuring this revision and exact base `${{ github.event.pull_request.base.sha }}` on the same runner. - name: Set Python to PATH uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -66,20 +70,34 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Ubuntu) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Install Conda + add to PATH (macOS) if: startsWith(matrix.os, 'macos') + env: + CONDA_ARCH: ${{ matrix.conda_arch }} run: | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - bash ~/miniconda.sh -b -p ~/miniconda - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV + if [[ "$(uname -m)" != "$CONDA_ARCH" ]]; then + echo "Conda installer architecture $CONDA_ARCH does not match runner $(uname -m)" >&2 + exit 1 + fi + curl --fail --location --show-error --output "$RUNNER_TEMP/miniconda.sh" \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh" + bash "$RUNNER_TEMP/miniconda.sh" -b -p "$HOME/miniconda" + echo "PATH=$PATH:$HOME/miniconda/bin" >> "$GITHUB_ENV" + echo "CONDA=$HOME/miniconda" >> "$GITHUB_ENV" shell: bash + - name: Check Conda version + run: conda info --all + - name: Create test Conda environment run: conda create -n perf-test-env python=3.12 -y @@ -93,26 +111,148 @@ jobs: toolchain: stable targets: ${{ matrix.target }} + - name: Prepare Exact Base Worktree + id: baseline + env: + PR_BASE: ${{ github.event.pull_request.base.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + base="$PR_BASE" + else + base="$(git rev-parse --verify "refs/remotes/origin/$DEFAULT_BRANCH^{commit}")" + fi + if [[ ! "$base" =~ ^[0-9a-f]{40}$ ]]; then + echo "Missing or invalid exact base revision" >&2 + exit 1 + fi + git worktree add --detach "$RUNNER_TEMP/perf-base" "$base" + mkdir -p baseline-perf + echo "sha=$base" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Capture Performance Measurement Context + id: measurement_context + env: + BASE_SHA: ${{ steps.baseline.outputs.sha }} + HEAD_SHA: ${{ github.sha }} + MEASUREMENT_TARGET: ${{ matrix.target }} + run: | + python - <<'PY' + import json + import os + import shutil + import subprocess + from pathlib import Path + + def output(*arguments, cwd=None): + executable = shutil.which(arguments[0]) + if executable is None: + raise RuntimeError(f"Required measurement tool not found: {arguments[0]}") + value = subprocess.check_output( + [executable, *arguments[1:]], cwd=cwd, text=True, encoding="utf-8" + ).strip() + if not value: + raise RuntimeError(f"Empty measurement version: {arguments[0]}") + return value + + workspace = Path(os.environ["GITHUB_WORKSPACE"]).resolve() + runner = {key: os.environ[key] for key in ( + "RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion" + )} + if not all(value.strip() for value in runner.values()): + raise RuntimeError("Runner image context must not be empty") + venv_python = ( + workspace / ".venv" / "Scripts" / "python.exe" if os.name == "nt" + else workspace / ".venv" / "bin" / "python" + ) + interpreters = { + "venv": output(str(venv_python), "--version"), + "conda-base": output("conda", "run", "--name", "base", "python", "--version"), + "conda-named": output("conda", "run", "--name", "perf-test-env", "python", "--version"), + } + for directory, destination, variable in ( + (workspace, workspace, "HEAD_SHA"), + (Path(os.environ["RUNNER_TEMP"]) / "perf-base", workspace / "baseline-perf", "BASE_SHA"), + ): + source = output("git", "rev-parse", "HEAD", cwd=directory) + if source != os.environ[variable]: + raise RuntimeError(f"Unexpected {variable} checkout: {source}") + context = { + "runner": runner, + "installed_target": os.environ["MEASUREMENT_TARGET"], + "features": "ci-perf", + "discovery_workspace": str(workspace), + "versions": { + "rustc": output("rustc", "-vV", cwd=directory), + "cargo": output("cargo", "--version", cwd=directory), + "python": output("python", "--version"), + "conda": output("conda", "--version"), + }, + "interpreters": interpreters, + } + (destination / "source-revision.txt").write_text(source + "\n", encoding="utf-8") + (destination / "measurement-context.json").write_text( + json.dumps(context, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + PY + shell: bash + - name: Cargo Fetch - run: cargo fetch + run: | + cargo fetch --locked + cd "$RUNNER_TEMP/perf-base" + cargo fetch --locked shell: bash - - name: Build Release - run: cargo build --release --target ${{ matrix.target }} + - name: Build Release and Benchmark Harnesses + id: build + env: + MEASUREMENT_TARGET: ${{ matrix.target }} + run: | + for source in "$RUNNER_TEMP/perf-base" "$GITHUB_WORKSPACE"; do + ( + cd "$source" + cargo build --frozen --release --target-dir target --target "$MEASUREMENT_TARGET" + cargo test --frozen --release --target-dir target --features ci-perf --target "$MEASUREMENT_TARGET" \ + --test e2e_performance --no-run + git diff --exit-code -- Cargo.toml Cargo.lock crates + ) + done + shell: bash + + - name: Run Exact Base Performance Tests + id: base_benchmark + env: + MEASUREMENT_TARGET: ${{ matrix.target }} + RUST_BACKTRACE: 1 + RUST_LOG: warn + run: | + set -o pipefail + cd "$RUNNER_TEMP/perf-base" + cargo test --frozen --release --target-dir target --features ci-perf --target "$MEASUREMENT_TARGET" \ + --test e2e_performance test_performance_summary -- --nocapture 2>&1 \ + | tee "$GITHUB_WORKSPACE/baseline-perf/perf-output.txt" shell: bash - name: Run Performance Tests id: benchmark + if: >- + always() && steps.build.outcome == 'success' && + steps.measurement_context.outcome == 'success' run: | set -o pipefail - cargo test --release --features ci-perf --target ${{ matrix.target }} --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt + cargo test --frozen --release --target-dir target --features ci-perf --target "$MEASUREMENT_TARGET" \ + --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt env: + MEASUREMENT_TARGET: ${{ matrix.target }} RUST_BACKTRACE: 1 RUST_LOG: warn shell: bash - name: Extract Performance Metrics - if: steps.benchmark.outcome == 'success' + id: metrics + if: always() && steps.benchmark.outcome == 'success' run: | if ! grep -q "JSON metrics:" perf-output.txt; then echo "Performance test produced no JSON metrics" >&2 @@ -123,56 +263,103 @@ jobs: cat metrics.json shell: bash - - name: Wait for Exact PR Base Performance - id: wait_for_base_performance - if: always() && github.event_name == 'pull_request' - run: >- - python scripts/wait_for_baseline.py - --repository "${{ github.repository }}" - --workflow perf-baseline.yml - --commit "${{ github.event.pull_request.base.sha }}" - --artifact "perf-baseline-${{ matrix.os }}" - --timeout-seconds 1200 - --poll-seconds 20 - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Extract Exact Base Performance Metrics + id: base_metrics + if: always() && steps.base_benchmark.outcome == 'success' + run: | + if ! grep -q "JSON metrics:" baseline-perf/perf-output.txt; then + echo "Exact base performance test produced no JSON metrics" >&2 + exit 1 + fi + sed -n '/JSON metrics:/,/^}/p' baseline-perf/perf-output.txt | tail -n +2 > baseline-perf/metrics.json + python -m json.tool baseline-perf/metrics.json > /dev/null + cat baseline-perf/metrics.json shell: bash - - name: Download Exact PR Base Performance - if: >- - always() && github.event_name == 'pull_request' && - steps.wait_for_base_performance.outcome == 'success' - uses: dawidd6/action-download-artifact@eab87c9830c39eff17e5a6eadb20bfb4bc880477 # v27 - with: - workflow: perf-baseline.yml - commit: ${{ github.event.pull_request.base.sha }} - workflow_conclusion: success - name: perf-baseline-${{ matrix.os }} - path: baseline-perf - check_artifacts: true - search_artifacts: true - - - name: Download Main Performance for Manual Run - if: always() && github.event_name == 'workflow_dispatch' - uses: dawidd6/action-download-artifact@eab87c9830c39eff17e5a6eadb20bfb4bc880477 # v27 - with: - workflow: perf-baseline.yml - branch: main - workflow_conclusion: success - name: perf-baseline-${{ matrix.os }} - path: baseline-perf - check_artifacts: true - search_artifacts: true - - name: Compare Performance Snapshot if: always() - run: >- - python scripts/quality_snapshot.py performance - --current metrics.json - --baseline baseline-perf/metrics.json - --platform "${{ matrix.platform }}" - --report performance-report.md - --summary "$GITHUB_STEP_SUMMARY" + env: + BASE_SHA: ${{ steps.baseline.outputs.sha }} + HEAD_SHA: ${{ github.sha }} + MEASUREMENT_TARGET: ${{ matrix.target }} + HEAD_BENCHMARK_OUTCOME: ${{ steps.benchmark.outcome }} + BASE_BENCHMARK_OUTCOME: ${{ steps.base_benchmark.outcome }} + HEAD_METRICS_OUTCOME: ${{ steps.metrics.outcome }} + BASE_METRICS_OUTCOME: ${{ steps.base_metrics.outcome }} + PLATFORM: ${{ matrix.platform }} + run: | + python - <<'PY' + import argparse + import os + import re + import sys + from pathlib import Path + + sys.path.insert(0, "scripts") + from quality_snapshot import SnapshotError, load_json, run_performance, write_report + + report = Path("performance-report.md") + summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) + platform = os.environ["PLATFORM"] + + def context(path): + value = load_json(path) + required = { + "runner": ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"), + "versions": ("rustc", "cargo", "python", "conda"), + "interpreters": ("conda-base", "conda-named", "venv"), + } + for group, keys in required.items(): + values = value.get(group) + if not isinstance(values, dict) or any( + not isinstance(values.get(key), str) or not values[key].strip() + for key in keys + ): + raise SnapshotError(f"Missing {group} measurement context in {path}") + if ( + value.get("features") != "ci-perf" + or value.get("installed_target") != os.environ["MEASUREMENT_TARGET"] + or value.get("discovery_workspace") != str(Path(os.environ["GITHUB_WORKSPACE"]).resolve()) + ): + raise SnapshotError(f"Incompatible performance workload in {path}") + return value + + try: + for variable in ( + "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", + "HEAD_METRICS_OUTCOME", "BASE_METRICS_OUTCOME", + ): + if os.environ.get(variable) != "success": + raise SnapshotError(f"Required benchmark or JSON extraction failed: {variable}") + for filename, variable in ( + ("source-revision.txt", "HEAD_SHA"), + ("baseline-perf/source-revision.txt", "BASE_SHA"), + ): + expected = os.environ.get(variable, "") + if ( + re.fullmatch(r"[0-9a-f]{40}", expected) is None + or Path(filename).read_text(encoding="utf-8").strip() != expected + ): + raise SnapshotError(f"Unexpected source revision in {filename}") + current = context(Path("measurement-context.json")) + baseline = context(Path("baseline-perf/measurement-context.json")) + if current != baseline: + raise SnapshotError( + "Performance runner, tool, or fixture versions differ; inspect both measurement contexts" + ) + except (OSError, UnicodeError, SnapshotError) as error: + print(f"Invalid performance measurement: {error}", file=sys.stderr) + write_report( + f"## Performance Report ({platform})\n\n:x: **Invalid snapshot:** {error}\n", + report, summary, + ) + raise SystemExit(1) + + raise SystemExit(run_performance(argparse.Namespace( + current=Path("metrics.json"), baseline=Path("baseline-perf/metrics.json"), + platform=platform, report=report, summary=summary, + ))) + PY shell: bash - name: Upload PR Performance Results @@ -182,6 +369,10 @@ jobs: name: perf-pr-${{ matrix.os }} path: | metrics.json + measurement-context.json + source-revision.txt + baseline-perf/ + perf-output.txt performance-report.md if-no-files-found: ignore diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index a916d2e4..ef421cc3 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -32,11 +32,13 @@ jobs: - os: ubuntu-latest target: x86_64-unknown-linux-musl run_cli: "yes" - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin + conda_arch: x86_64 run_cli: "yes" - - os: macos-14 + - os: macos-15 target: aarch64-apple-darwin + conda_arch: arm64 run_cli: "yes" steps: - name: Checkout @@ -58,18 +60,29 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH + - name: Set up Conda (Linux) if: startsWith( matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Install Conda + add to PATH if: startsWith( matrix.os, 'macos') + env: + CONDA_ARCH: ${{ matrix.conda_arch }} run: | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - bash ~/miniconda.sh -b -p ~/miniconda - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV + if [[ "$(uname -m)" != "$CONDA_ARCH" ]]; then + echo "Conda installer architecture $CONDA_ARCH does not match runner $(uname -m)" >&2 + exit 1 + fi + curl --fail --location --show-error --output "$RUNNER_TEMP/miniconda.sh" \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh" + bash "$RUNNER_TEMP/miniconda.sh" -b -p "$HOME/miniconda" + echo "PATH=$PATH:$HOME/miniconda/bin" >> "$GITHUB_ENV" + echo "CONDA=$HOME/miniconda" >> "$GITHUB_ENV" shell: bash - name: Check Conda version @@ -459,10 +472,10 @@ jobs: # target: aarch64-unknown-linux-gnu # - os: ubuntu-latest # target: arm-unknown-linux-gnueabihf - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin run_cli: "yes" - - os: macos-14 + - os: macos-15 target: aarch64-apple-darwin run_cli: "yes" # - os: ubuntu-latest diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index df35bed8..bfe4d8d8 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -1,6 +1,45 @@ # Quality snapshots -PET uses pull-request snapshots to prevent performance and coverage drift. Each pull request is compared with artifacts produced for the exact pull-request base commit, not the latest moving `main` tip. +PET uses pull-request snapshots to prevent performance and coverage drift. Each pull request +measures its exact base commit with the current workflow setup, rather than compare against a +moving `main` tip or a historical measurement from a different compiler or runner image. + +## Comparable measurements + +Linux and Windows coverage run the exact base and tested PR revision in separate fresh jobs. +Both use the same installed-manager `ci` workload and fixture setup. They must not be measured +sequentially against shared mutable fixtures: the Unix Conda CI tests create environments and +install Mamba into the base installation. Native ARM64 macOS coverage retains its separate +default-feature workload and same-runner base measurement. + +Coverage measurement artifacts include `source-revision.txt` and `measurement-context.json`, +LCOV, production reports, and subprocess proof. Artifact names distinguish revision and workflow +attempt; partial reruns cannot silently reuse the other revision's earlier measurement. The gate +requires successful measurements/proofs, the expected source SHAs, and matching nonempty runner +image, architecture, toolchain/LLVM, tool, feature, and fixture-interpreter versions before the +unchanged numerical comparison. Missing or incompatible evidence is an invalid snapshot, not an +improvement or a reason to fall back to an older successful artifact. Separate coverage machines +are not claimed to have identical hardware; observable context mismatches remain blocking. + +Performance builds each revision and its benchmark harness in its own source tree before timing, +then measures the exact base and PR sequentially on the same runner. The detached base worktree +is outside the discovery workspace; both harnesses inherit the same `GITHUB_WORKSPACE`, PATH, +Conda installation, and workspace `.venv`. Builds explicitly use each source tree's normal +`target` directory because the harness resolves its PET executable there rather than honor +`CARGO_TARGET_DIR`. Each invocation still uses independent PID-specific caches and retains every +cold/warm sample and inventory check. Both benchmark executions and JSON extractions must succeed; +the report retains both actual snapshots, source/context evidence, and benchmark logs. +Manual performance runs resolve the default branch's exact SHA once and measure it the same way. + +Linux fixture jobs explicitly provision Miniconda without creating or activating an additional +environment. macOS x86_64 installed-manager/performance jobs use `macos-15-intel`; existing ARM64 +jobs use `macos-15`, with matching native Conda installers. Host-image inventory changes are +measured on both revisions, never normalized away or accepted by changing inventory schemas. + +The push/manual coverage and performance baseline workflows remain publishers for inspection +and history, but PR gates no longer depend on their historical artifacts. Fresh comparisons add +a real base coverage job and base performance build/measurement per platform. All regression +budgets, sample requirements, inventory checks, and required coverage evidence remain unchanged. ## Performance gate diff --git a/scripts/tests/test_quality_workflows.py b/scripts/tests/test_quality_workflows.py index c467f5eb..4b524734 100644 --- a/scripts/tests/test_quality_workflows.py +++ b/scripts/tests/test_quality_workflows.py @@ -1,10 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import copy +import json +import os import re +import subprocess +import sys +import tempfile +import textwrap import unittest from pathlib import Path +from test_quality_snapshot import performance_snapshot, write_lcov + ROOT = Path(__file__).resolve().parents[2] SAME_REPOSITORY_PR = ( @@ -38,7 +47,117 @@ def step_property(step: str, name: str) -> str | None: return None +def workflow_python(step: str) -> str: + source = step.split(" python - <<'PY'\n", 1)[1] + return textwrap.dedent(source.split("\n PY", 1)[0]) + + class QualityWorkflowTests(unittest.TestCase): + def test_linux_conda_setup_does_not_create_or_activate_an_extra_environment(self) -> None: + for workflow in ( + "pr-check.yml", "coverage.yml", "coverage-baseline.yml", + "perf-tests.yml", "perf-baseline.yml", + ): + with self.subTest(workflow=workflow): + path = ".github/workflows/" + workflow + steps = workflow_steps(path) + setup = steps["Set up Conda (Linux)"] + self.assertRegex(setup, r"conda-incubator/setup-miniconda@[0-9a-f]{40}\b") + for setting in ( + 'activate-environment: ""', "auto-activate: false", "run-init: false", + ): + self.assertIn(setting, setup) + self.assertNotIn("$CONDA/condabin", (ROOT / path).read_text(encoding="utf-8")) + self.assertIn("conda info --all", steps["Check Conda version"]) + + def test_macos_runners_keep_both_native_architecture_workloads(self) -> None: + expected = { + "pr-check.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ] * 2, + "build.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ], + "perf-tests.yml": [("macos-15-intel", "x86_64-apple-darwin")], + "perf-baseline.yml": [("macos-15-intel", "x86_64-apple-darwin")], + } + for workflow, targets in expected.items(): + with self.subTest(workflow=workflow): + text = (ROOT / ".github" / "workflows" / workflow).read_text(encoding="utf-8") + self.assertEqual( + re.findall(r"- os: (macos-\S+)\n\s+target: (\S+)", text), + targets, + ) + self.assertNotIn("macos-14", text) + if workflow != "build.yml": + self.assertIn("CONDA_ARCH: ${{ matrix.conda_arch }}", text) + self.assertIn('if [[ "$(uname -m)" != "$CONDA_ARCH" ]]', text) + self.assertIn("Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh", text) + coverage = (ROOT / ".github/workflows/coverage-macos.yml").read_text(encoding="utf-8") + self.assertIn("runs-on: macos-15\n", coverage) + self.assertNotIn("macos-14", coverage) + + def test_coverage_pairs_clean_exact_revision_jobs_from_the_same_attempt(self) -> None: + path = ".github/workflows/coverage.yml" + text = (ROOT / path).read_text(encoding="utf-8") + steps = workflow_steps(path) + self.assertIn("revision: [base, head]", text) + self.assertIn( + "ref: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }}", + text, + ) + self.assertIn("needs: measurements", text) + self.assertIn("MEASUREMENTS_RESULT: ${{ needs.measurements.result }}", text) + self.assertIn('prefix="$(pipenv --venv)"', steps["Create a Pipenv Environment"]) + self.assertLess( + text.index("- name: Create a Pipenv Environment"), + text.index("- name: Create .venv"), + ) + self.assertIn( + "CREATED_PIPENV_PREFIX: ${{ steps.pipenv.outputs.prefix }}", + steps["Capture Coverage Measurement Context"], + ) + upload = steps["Upload Coverage Measurement"] + self.assertIn("${{ matrix.revision }}-${{ github.run_attempt }}", upload) + for step, revision in ( + ("Download Head Coverage Measurement", "head"), + ("Download Exact PR Base Coverage", "base"), + ): + self.assertIn(f"-{revision}-${{{{ github.run_attempt }}}}", steps[step]) + self.assertNotIn("wait_for_baseline.py", text) + self.assertNotIn("dawidd6/action-download-artifact", text) + + def test_performance_builds_each_harness_before_timing_a_shared_workspace(self) -> None: + path = ".github/workflows/perf-tests.yml" + text = (ROOT / path).read_text(encoding="utf-8") + steps = workflow_steps(path) + prepare = steps["Prepare Exact Base Worktree"] + self.assertIn('git worktree add --detach "$RUNNER_TEMP/perf-base" "$base"', prepare) + self.assertIn('base="$PR_BASE"', prepare) + self.assertIn("refs/remotes/origin/$DEFAULT_BRANCH", prepare) + build = steps["Build Release and Benchmark Harnesses"] + self.assertIn('for source in "$RUNNER_TEMP/perf-base" "$GITHUB_WORKSPACE"', build) + self.assertIn("--no-run", build) + for step in (build, steps["Run Exact Base Performance Tests"], steps["Run Performance Tests"]): + self.assertIn("--target-dir target", step) + self.assertIn("--features ci-perf", step) + self.assertLess( + text.index("- name: Build Release and Benchmark Harnesses"), + text.index("- name: Run Exact Base Performance Tests"), + ) + self.assertIn('cd "$RUNNER_TEMP/perf-base"', steps["Run Exact Base Performance Tests"]) + self.assertNotIn("CARGO_TARGET_DIR:", text) + self.assertNotIn("GITHUB_WORKSPACE:", text) + self.assertNotIn("wait_for_baseline.py", text) + self.assertNotIn("dawidd6/action-download-artifact", text) + for step in ("Extract Performance Metrics", "Extract Exact Base Performance Metrics"): + self.assertIn("always()", step_property(steps[step], "if")) + self.assertIn("python -m json.tool", steps[step]) + for artifact in ("baseline-perf/", "measurement-context.json", "source-revision.txt", "perf-output.txt"): + self.assertIn(artifact, steps["Upload PR Performance Results"]) + def test_comment_steps_are_fork_safe_and_non_gating(self) -> None: comment_steps = [] for path in (ROOT / ".github" / "workflows").glob("*.yml"): @@ -130,5 +249,135 @@ def test_report_artifacts_are_uploaded_after_comparison(self) -> None: self.assertIsNone(step_property(steps[comparison], "continue-on-error")) +class MeasurementComparisonTests(unittest.TestCase): + def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None): + baseline_name = "baseline-coverage" if kind == "coverage" else "baseline-perf" + workflow = "coverage.yml" if kind == "coverage" else "perf-tests.yml" + step_name = "Compare Coverage Snapshot" if kind == "coverage" else "Compare Performance Snapshot" + code = workflow_python(workflow_steps(".github/workflows/" + workflow)[step_name]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + baseline = root / baseline_name + baseline.mkdir() + context = { + "runner": dict.fromkeys( + ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"), "runner-version" + ), + "installed_target": "x86_64-unknown-linux-musl", + "features": "ci" if kind == "coverage" else "ci-perf", + "versions": dict.fromkeys(("rustc", "cargo", "python", "conda"), "tool-version"), + "interpreters": dict.fromkeys( + ("conda-base", "conda-named", "venv"), "Python 3.12.1" + ), + } + if kind == "coverage": + context["coverage_target"] = "host" + context["versions"].update(dict.fromkeys( + ("cargo-llvm-cov", "pipenv", "pyenv", "pixi"), "tool-version" + )) + context["interpreters"].update(dict.fromkeys( + ("conda-prefix", "venv2", "pipenv", "pixi-default", "pixi-dev", "pyenv"), + "Python 3.12.1", + )) + for path, hits in ((root, 8000 if regression else 9000), (baseline, 9000)): + write_lcov( + path / "lcov.info", lines_hit=hits, lines_found=10000, + functions_hit=900, functions_found=1000, + ) + else: + context["discovery_workspace"] = str(root.resolve()) + for path, startup in ((root, 5000 if regression else 20), (baseline, 20)): + snapshot = performance_snapshot( + schema_version=3, inventory_schema_version=2, startup_p95=startup, + ) + (path / "metrics.json").write_text(json.dumps(snapshot), encoding="utf-8") + current_context = copy.deepcopy(context) + base_context = copy.deepcopy(context) + if invalid == "tool": + base_context["versions"]["rustc"] = "different compiler" + elif invalid == "image": + base_context["runner"]["ImageVersion"] = "different runner image" + elif invalid == "fixture": + base_context["interpreters"]["venv"] = "Python 3.13.1" + elif invalid == "incomplete": + del current_context["versions"]["rustc"] + del base_context["versions"]["rustc"] + elif invalid == "workload": + current_context["features"] = base_context["features"] = "other-features" + elif invalid not in (None, "missing", "malformed", "revision"): + raise ValueError(f"Unknown invalid measurement case: {invalid}") + (root / "measurement-context.json").write_text(json.dumps(current_context), encoding="utf-8") + (baseline / "measurement-context.json").write_text(json.dumps(base_context), encoding="utf-8") + (root / "source-revision.txt").write_text("a" * 40 + "\n", encoding="utf-8") + (baseline / "source-revision.txt").write_text( + ("a" if invalid == "revision" else "b") * 40 + "\n", encoding="utf-8" + ) + if invalid == "missing": + (baseline / "measurement-context.json").unlink() + elif invalid == "malformed": + (baseline / "measurement-context.json").write_text("{", encoding="utf-8") + environment = { + **os.environ, + "PYTHONPATH": str(ROOT / "scripts"), + "PYTHONDONTWRITEBYTECODE": "1", + "GITHUB_WORKSPACE": str(root), + "GITHUB_STEP_SUMMARY": str(root / "summary.md"), + "PLATFORM": "Linux", + "MEASUREMENT_TARGET": "x86_64-unknown-linux-musl", + "HEAD_SHA": "a" * 40, + "BASE_SHA": "b" * 40, + "MEASUREMENTS_RESULT": "success", + **dict.fromkeys(( + "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", + "HEAD_METRICS_OUTCOME", "BASE_METRICS_OUTCOME", + ), "success"), + **(outcomes or {}), + } + result = subprocess.run( + [sys.executable, "-B", "-c", code], cwd=root, env=environment, + capture_output=True, text=True, check=False, + ) + report_path = root / ( + "coverage-report.md" if kind == "coverage" else "performance-report.md" + ) + self.assertTrue(report_path.is_file(), result.stderr) + return result, report_path.read_text(encoding="utf-8") + + def test_complete_measurements_retain_the_numerical_regression_gates(self) -> None: + for kind in ("coverage", "performance"): + for regression in (False, True): + with self.subTest(kind=kind, regression=regression): + result, report = self.run_measurement(kind, regression=regression) + self.assertEqual(result.returncode, int(regression), result.stderr) + self.assertNotIn("Invalid snapshot", report) + self.assertIn( + "Regression detected" if regression else "Within regression budget", report + ) + + def test_invalid_provenance_or_context_cannot_pass_unchanged_snapshots(self) -> None: + for kind in ("coverage", "performance"): + for invalid in ( + "missing", "malformed", "incomplete", "tool", + "image", "fixture", "workload", "revision", + ): + with self.subTest(kind=kind, invalid=invalid): + result, report = self.run_measurement(kind, invalid=invalid) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Invalid snapshot", report) + self.assertNotIn("Within regression budget", report) + + def test_failed_execution_or_extraction_cannot_pass_complete_snapshots(self) -> None: + for variable in ( + "MEASUREMENTS_RESULT", "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", + "HEAD_METRICS_OUTCOME", "BASE_METRICS_OUTCOME", + ): + kind = "coverage" if variable == "MEASUREMENTS_RESULT" else "performance" + with self.subTest(variable=variable): + result, report = self.run_measurement(kind, outcomes={variable: "failure"}) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Invalid snapshot", report) + self.assertNotIn("Within regression budget", report) + + if __name__ == "__main__": unittest.main() From f10594fd644f112fa609dce548feab29d0dd4c92 Mon Sep 17 00:00:00 2001 From: Stella Huang Date: Mon, 16 Nov 2026 13:49:58 -0800 Subject: [PATCH 2/2] ci: extract measurement helpers and test ARM64 perf Move shared measurement capture and provenance checks into an argument-driven script while retaining workload-specific validation. Stage the tested helper revision outside the discovery tree before checking out old coverage source. Retain Intel performance and add a native ARM64 pair to PR and baseline matrices without changing quality budgets. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/coverage.yml | 196 ++++----------- .github/workflows/perf-baseline.yml | 3 + .github/workflows/perf-tests.yml | 167 +++---------- docs/QUALITY_SNAPSHOTS.md | 21 +- scripts/quality_measurement.py | 314 ++++++++++++++++++++++++ scripts/tests/test_quality_workflows.py | 235 ++++++++++++++++-- 6 files changed, 630 insertions(+), 306 deletions(-) create mode 100644 scripts/quality_measurement.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c774efca..a5acb01a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -32,10 +32,10 @@ jobs: platform: Windows comment_header: coverage-windows steps: - - name: Checkout + - name: Checkout Workflow uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }} + ref: ${{ github.sha }} fetch-depth: 0 persist-credentials: false @@ -63,6 +63,21 @@ jobs: run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v shell: bash + - name: Stage Coverage Measurement Helper + run: >- + python scripts/quality_measurement.py stage + --source-dir "$GITHUB_WORKSPACE" + --output-dir "$RUNNER_TEMP/quality-tools" + --revision "$GITHUB_SHA" + shell: bash + + - name: Checkout Measurement Source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Add Conda to PATH (Windows) if: startsWith(matrix.os, 'windows') run: | @@ -214,80 +229,17 @@ jobs: - name: Capture Coverage Measurement Context env: MEASUREMENT_TARGET: ${{ matrix.target }} + MEASURED_SHA: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }} CREATED_PIPENV_PREFIX: ${{ steps.pipenv.outputs.prefix }} - run: | - git rev-parse HEAD > source-revision.txt - python - <<'PY' - import json - import os - import shutil - import subprocess - from pathlib import Path - - def output(*arguments): - executable = shutil.which(arguments[0]) - if executable is None: - raise RuntimeError(f"Required measurement tool not found: {arguments[0]}") - value = subprocess.check_output( - [executable, *arguments[1:]], text=True, encoding="utf-8" - ).strip() - if not value: - raise RuntimeError(f"Empty measurement version: {arguments[0]}") - return value - - runner = {key: os.environ[key] for key in ( - "RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion" - )} - if not all(value.strip() for value in runner.values()): - raise RuntimeError("Runner image context must not be empty") - windows = os.name == "nt" - python_name = "python.exe" if windows else "python" - venv_bin = "Scripts" if windows else "bin" - pipenv_prefix = Path(os.environ["CREATED_PIPENV_PREFIX"]) - interpreters = { - name: output(str(prefix / venv_bin / python_name), "--version") - for name, prefix in ( - ("venv", Path(".venv")), - ("venv2", Path(".venv2")), - ("pipenv", pipenv_prefix), - ) - } - for name in ("default", "dev"): - prefix = Path(".pixi") / "envs" / name - executable = prefix / python_name if windows else prefix / "bin" / python_name - interpreters[f"pixi-{name}"] = output(str(executable), "--version") - for label, selector in ( - ("base", ("--name", "base")), - ("named", ("--name", "test-env1")), - ("prefix", ("--prefix", str(Path("prefix-envs/.conda1").resolve()))), - ): - interpreters[f"conda-{label}"] = output( - "conda", "run", *selector, "python", "--version" - ) - interpreters["pyenv"] = "\n".join(sorted( - output("pyenv", "versions", "--bare").splitlines() - )) - context = { - "runner": runner, - "installed_target": os.environ["MEASUREMENT_TARGET"], - "coverage_target": "host", - "features": "ci", - "versions": { - "rustc": output("rustc", "-vV"), - "cargo": output("cargo", "--version"), - "cargo-llvm-cov": output("cargo", "llvm-cov", "--version"), - "python": output("python", "--version"), - "conda": output("conda", "--version"), - "pipenv": output("pipenv", "--version"), - "pyenv": output("pyenv", "--version"), - "pixi": output("pixi", "--version"), - }, - "interpreters": interpreters, - } - Path("measurement-context.json").write_text( - json.dumps(context, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - PY + run: >- + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" capture coverage + --source-dir "$GITHUB_WORKSPACE" + --workspace "$GITHUB_WORKSPACE" + --output-dir "$GITHUB_WORKSPACE" + --target "$MEASUREMENT_TARGET" + --expected-source "$MEASURED_SHA" + --workflow-revision "$GITHUB_SHA" + --pipenv-prefix "$CREATED_PIPENV_PREFIX" shell: bash - name: Run Tests with Coverage @@ -355,6 +307,14 @@ jobs: with: python-version: "3.12" + - name: Stage Coverage Comparison Helper + run: >- + python scripts/quality_measurement.py stage + --source-dir "$GITHUB_WORKSPACE" + --output-dir "$RUNNER_TEMP/quality-tools" + --revision "$GITHUB_SHA" + shell: bash + - name: Download Head Coverage Measurement uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -375,81 +335,17 @@ jobs: MEASUREMENT_TARGET: ${{ matrix.target }} MEASUREMENTS_RESULT: ${{ needs.measurements.result }} PLATFORM: ${{ matrix.platform }} - run: | - python - <<'PY' - import argparse - import os - import re - import sys - from pathlib import Path - - sys.path.insert(0, "scripts") - from quality_snapshot import SnapshotError, load_json, run_coverage, write_report - - report = Path("coverage-report.md") - summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) - platform = os.environ["PLATFORM"] - - def context(path): - value = load_json(path) - required = { - "runner": ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"), - "versions": ( - "rustc", "cargo", "cargo-llvm-cov", "python", - "conda", "pipenv", "pyenv", "pixi", - ), - "interpreters": ( - "conda-base", "conda-named", "conda-prefix", "venv", "venv2", - "pipenv", "pixi-default", "pixi-dev", "pyenv", - ), - } - for group, keys in required.items(): - values = value.get(group) - if not isinstance(values, dict) or any( - not isinstance(values.get(key), str) or not values[key].strip() - for key in keys - ): - raise SnapshotError(f"Missing {group} measurement context in {path}") - if ( - value.get("features") != "ci" - or value.get("coverage_target") != "host" - or value.get("installed_target") != os.environ["MEASUREMENT_TARGET"] - ): - raise SnapshotError(f"Incompatible coverage workload in {path}") - return value - - try: - if os.environ.get("MEASUREMENTS_RESULT") != "success": - raise SnapshotError("One or more coverage measurements or required proofs failed") - for filename, variable in ( - ("source-revision.txt", "HEAD_SHA"), - ("baseline-coverage/source-revision.txt", "BASE_SHA"), - ): - expected = os.environ.get(variable, "") - if ( - re.fullmatch(r"[0-9a-f]{40}", expected) is None - or Path(filename).read_text(encoding="utf-8").strip() != expected - ): - raise SnapshotError(f"Unexpected source revision in {filename}") - current = context(Path("measurement-context.json")) - baseline = context(Path("baseline-coverage/measurement-context.json")) - if current != baseline: - raise SnapshotError( - "Coverage runner, tool, or fixture versions differ; inspect both measurement contexts" - ) - except (OSError, UnicodeError, SnapshotError) as error: - print(f"Invalid coverage measurement: {error}", file=sys.stderr) - write_report( - f"## Test Coverage Report ({platform})\n\n:x: **Invalid snapshot:** {error}\n", - report, summary, - ) - raise SystemExit(1) - - raise SystemExit(run_coverage(argparse.Namespace( - current=Path("lcov.info"), baseline=Path("baseline-coverage/lcov.info"), - platform=platform, report=report, summary=summary, - ))) - PY + run: >- + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" compare coverage + --current-dir . + --baseline-dir baseline-coverage + --expected-head "$HEAD_SHA" + --expected-base "$BASE_SHA" + --target "$MEASUREMENT_TARGET" + --measurements-result "$MEASUREMENTS_RESULT" + --platform "$PLATFORM" + --report coverage-report.md + --summary "$GITHUB_STEP_SUMMARY" shell: bash - name: Upload PR Coverage Artifact diff --git a/.github/workflows/perf-baseline.yml b/.github/workflows/perf-baseline.yml index 7830f80b..b91bf870 100644 --- a/.github/workflows/perf-baseline.yml +++ b/.github/workflows/perf-baseline.yml @@ -27,6 +27,9 @@ jobs: - os: macos-15-intel target: x86_64-apple-darwin conda_arch: x86_64 + - os: macos-15 + target: aarch64-apple-darwin + conda_arch: arm64 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/perf-tests.yml b/.github/workflows/perf-tests.yml index 703c3868..ec0cccbe 100644 --- a/.github/workflows/perf-tests.yml +++ b/.github/workflows/perf-tests.yml @@ -35,6 +35,11 @@ jobs: conda_arch: x86_64 platform: macOS comment_header: perf-macos + - os: macos-15 + target: aarch64-apple-darwin + conda_arch: arm64 + platform: macOS ARM64 + comment_header: perf-macos-arm64 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -64,6 +69,14 @@ jobs: run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v shell: bash + - name: Stage Performance Measurement Helper + run: >- + python scripts/quality_measurement.py stage + --source-dir "$GITHUB_WORKSPACE" + --output-dir "$RUNNER_TEMP/quality-tools" + --revision "$GITHUB_SHA" + shell: bash + - name: Add Conda to PATH (Windows) if: startsWith(matrix.os, 'windows') run: | @@ -138,64 +151,14 @@ jobs: HEAD_SHA: ${{ github.sha }} MEASUREMENT_TARGET: ${{ matrix.target }} run: | - python - <<'PY' - import json - import os - import shutil - import subprocess - from pathlib import Path - - def output(*arguments, cwd=None): - executable = shutil.which(arguments[0]) - if executable is None: - raise RuntimeError(f"Required measurement tool not found: {arguments[0]}") - value = subprocess.check_output( - [executable, *arguments[1:]], cwd=cwd, text=True, encoding="utf-8" - ).strip() - if not value: - raise RuntimeError(f"Empty measurement version: {arguments[0]}") - return value - - workspace = Path(os.environ["GITHUB_WORKSPACE"]).resolve() - runner = {key: os.environ[key] for key in ( - "RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion" - )} - if not all(value.strip() for value in runner.values()): - raise RuntimeError("Runner image context must not be empty") - venv_python = ( - workspace / ".venv" / "Scripts" / "python.exe" if os.name == "nt" - else workspace / ".venv" / "bin" / "python" - ) - interpreters = { - "venv": output(str(venv_python), "--version"), - "conda-base": output("conda", "run", "--name", "base", "python", "--version"), - "conda-named": output("conda", "run", "--name", "perf-test-env", "python", "--version"), - } - for directory, destination, variable in ( - (workspace, workspace, "HEAD_SHA"), - (Path(os.environ["RUNNER_TEMP"]) / "perf-base", workspace / "baseline-perf", "BASE_SHA"), - ): - source = output("git", "rev-parse", "HEAD", cwd=directory) - if source != os.environ[variable]: - raise RuntimeError(f"Unexpected {variable} checkout: {source}") - context = { - "runner": runner, - "installed_target": os.environ["MEASUREMENT_TARGET"], - "features": "ci-perf", - "discovery_workspace": str(workspace), - "versions": { - "rustc": output("rustc", "-vV", cwd=directory), - "cargo": output("cargo", "--version", cwd=directory), - "python": output("python", "--version"), - "conda": output("conda", "--version"), - }, - "interpreters": interpreters, - } - (destination / "source-revision.txt").write_text(source + "\n", encoding="utf-8") - (destination / "measurement-context.json").write_text( - json.dumps(context, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - PY + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" capture performance \ + --source-dir "$GITHUB_WORKSPACE" --workspace "$GITHUB_WORKSPACE" \ + --output-dir "$GITHUB_WORKSPACE" --target "$MEASUREMENT_TARGET" \ + --expected-source "$HEAD_SHA" --workflow-revision "$GITHUB_SHA" + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" capture performance \ + --source-dir "$RUNNER_TEMP/perf-base" --workspace "$GITHUB_WORKSPACE" \ + --output-dir "$GITHUB_WORKSPACE/baseline-perf" --target "$MEASUREMENT_TARGET" \ + --expected-source "$BASE_SHA" --workflow-revision "$GITHUB_SHA" shell: bash - name: Cargo Fetch @@ -287,79 +250,21 @@ jobs: HEAD_METRICS_OUTCOME: ${{ steps.metrics.outcome }} BASE_METRICS_OUTCOME: ${{ steps.base_metrics.outcome }} PLATFORM: ${{ matrix.platform }} - run: | - python - <<'PY' - import argparse - import os - import re - import sys - from pathlib import Path - - sys.path.insert(0, "scripts") - from quality_snapshot import SnapshotError, load_json, run_performance, write_report - - report = Path("performance-report.md") - summary = Path(os.environ["GITHUB_STEP_SUMMARY"]) - platform = os.environ["PLATFORM"] - - def context(path): - value = load_json(path) - required = { - "runner": ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"), - "versions": ("rustc", "cargo", "python", "conda"), - "interpreters": ("conda-base", "conda-named", "venv"), - } - for group, keys in required.items(): - values = value.get(group) - if not isinstance(values, dict) or any( - not isinstance(values.get(key), str) or not values[key].strip() - for key in keys - ): - raise SnapshotError(f"Missing {group} measurement context in {path}") - if ( - value.get("features") != "ci-perf" - or value.get("installed_target") != os.environ["MEASUREMENT_TARGET"] - or value.get("discovery_workspace") != str(Path(os.environ["GITHUB_WORKSPACE"]).resolve()) - ): - raise SnapshotError(f"Incompatible performance workload in {path}") - return value - - try: - for variable in ( - "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", - "HEAD_METRICS_OUTCOME", "BASE_METRICS_OUTCOME", - ): - if os.environ.get(variable) != "success": - raise SnapshotError(f"Required benchmark or JSON extraction failed: {variable}") - for filename, variable in ( - ("source-revision.txt", "HEAD_SHA"), - ("baseline-perf/source-revision.txt", "BASE_SHA"), - ): - expected = os.environ.get(variable, "") - if ( - re.fullmatch(r"[0-9a-f]{40}", expected) is None - or Path(filename).read_text(encoding="utf-8").strip() != expected - ): - raise SnapshotError(f"Unexpected source revision in {filename}") - current = context(Path("measurement-context.json")) - baseline = context(Path("baseline-perf/measurement-context.json")) - if current != baseline: - raise SnapshotError( - "Performance runner, tool, or fixture versions differ; inspect both measurement contexts" - ) - except (OSError, UnicodeError, SnapshotError) as error: - print(f"Invalid performance measurement: {error}", file=sys.stderr) - write_report( - f"## Performance Report ({platform})\n\n:x: **Invalid snapshot:** {error}\n", - report, summary, - ) - raise SystemExit(1) - - raise SystemExit(run_performance(argparse.Namespace( - current=Path("metrics.json"), baseline=Path("baseline-perf/metrics.json"), - platform=platform, report=report, summary=summary, - ))) - PY + run: >- + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" compare performance + --current-dir . + --baseline-dir baseline-perf + --expected-head "$HEAD_SHA" + --expected-base "$BASE_SHA" + --target "$MEASUREMENT_TARGET" + --workspace "$GITHUB_WORKSPACE" + --head-benchmark-outcome "$HEAD_BENCHMARK_OUTCOME" + --base-benchmark-outcome "$BASE_BENCHMARK_OUTCOME" + --head-metrics-outcome "$HEAD_METRICS_OUTCOME" + --base-metrics-outcome "$BASE_METRICS_OUTCOME" + --platform "$PLATFORM" + --report performance-report.md + --summary "$GITHUB_STEP_SUMMARY" shell: bash - name: Upload PR Performance Results diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index bfe4d8d8..86f6c435 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -12,6 +12,18 @@ sequentially against shared mutable fixtures: the Unix Conda CI tests create env install Mamba into the base installation. Native ARM64 macOS coverage retains its separate default-feature workload and same-runner base measurement. +`scripts/quality_measurement.py` provides argument-driven staging, context capture, and +provenance validation for both workloads, delegating numerical gates to `quality_snapshot.py`. +Coverage still requires its additional llvm-cov/Pipenv/Pyenv/Pixi tools and fixture interpreters; +performance uses its own smaller context and shared discovery workspace. +Before selecting the measured coverage revision, the job stages only the tested workflow's +two helper script blobs and a revision marker outside the discovery tree in `RUNNER_TEMP`. +It then checks out the exact base or tested head and provisions fixtures there. An old base need +not contain the new helper: `source-revision.txt` identifies the measured Rust checkout, while +`helper_revision` records the separately staged control code and must match the tested workflow. +Missing helpers, marker, or provenance fail closed; this does not copy head Rust source or binaries +into the base measurement, or introduce another discoverable environment. + Coverage measurement artifacts include `source-revision.txt` and `measurement-context.json`, LCOV, production reports, and subprocess proof. Artifact names distinguish revision and workflow attempt; partial reruns cannot silently reuse the other revision's earlier measurement. The gate @@ -32,8 +44,11 @@ the report retains both actual snapshots, source/context evidence, and benchmark Manual performance runs resolve the default branch's exact SHA once and measure it the same way. Linux fixture jobs explicitly provision Miniconda without creating or activating an additional -environment. macOS x86_64 installed-manager/performance jobs use `macos-15-intel`; existing ARM64 -jobs use `macos-15`, with matching native Conda installers. Host-image inventory changes are +environment. macOS x86_64 installed-manager/performance jobs use `macos-15-intel`; ARM64 +jobs use `macos-15`, with matching native Conda installers. Both PR and baseline performance +matrices retain Intel and include a separate native ARM64 lane. Each architecture measures its +own exact base, uses distinct artifacts/comments, and retains the existing macOS budgets. +Host-image inventory changes are measured on both revisions, never normalized away or accepted by changing inventory schemas. The push/manual coverage and performance baseline workflows remain publishers for inspection @@ -44,7 +59,7 @@ budgets, sample requirements, inventory checks, and required coverage evidence r ## Performance gate The performance workflow runs 10 paired cache-cold/cache-warm JSON-RPC iterations on Linux, -Windows, and macOS, plus 10 untimed cache-cold diagnostic iterations. A comparison is valid only +Windows, Intel macOS, and ARM64 macOS, plus 10 untimed cache-cold diagnostic iterations. A comparison is valid only when every required distribution has at least five samples, inventories match within the same inventory schema, and both benchmark execution and JSON extraction succeed. diff --git a/scripts/quality_measurement.py b/scripts/quality_measurement.py new file mode 100644 index 00000000..d8f87ece --- /dev/null +++ b/scripts/quality_measurement.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Stage, capture, and validate comparable exact-base quality measurements.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Sequence + +if not Path(__file__).with_name("quality_snapshot.py").is_file(): + raise SystemExit("Missing measurement helper dependency: quality_snapshot.py") + +from quality_snapshot import ( # noqa: E402 + SnapshotError, + load_json, + run_coverage, + run_performance, + write_report, +) + + +RUNNER_FIELDS = ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion") +COMMON_TOOLS = { + "rustc": ("rustc", "-vV"), + "cargo": ("cargo", "--version"), + "python": ("python", "--version"), + "conda": ("conda", "--version"), +} +COVERAGE_TOOLS = { + "cargo-llvm-cov": ("cargo", "llvm-cov", "--version"), + "pipenv": ("pipenv", "--version"), + "pyenv": ("pyenv", "--version"), + "pixi": ("pixi", "--version"), +} +COMMON_INTERPRETERS = ("conda-base", "conda-named", "venv") +COVERAGE_INTERPRETERS = ( + "conda-prefix", "venv2", "pipenv", "pixi-default", "pixi-dev", "pyenv", +) +HELPER_FILES = ("quality_measurement.py", "quality_snapshot.py") +HELPER_REVISION = "helper-revision.txt" + + +def require_revision(value: str, description: str) -> str: + if re.fullmatch(r"[0-9a-f]{40}", value) is None: + raise SnapshotError(f"Missing or invalid {description}") + return value + + +def command_output(*arguments: str, cwd: Path) -> str: + executable = shutil.which(arguments[0]) + if executable is None: + raise SnapshotError(f"Required measurement tool not found: {arguments[0]}") + try: + value = subprocess.check_output( + [executable, *arguments[1:]], cwd=cwd, text=True, encoding="utf-8", + stderr=subprocess.PIPE, + ).strip() + except subprocess.CalledProcessError as error: + raise SnapshotError( + f"Measurement command failed ({arguments[0]}): {error.stderr.strip()}" + ) from error + if not value: + raise SnapshotError(f"Empty measurement output: {arguments[0]}") + return value + + +def source_revision(directory: Path, expected: str) -> str: + require_revision(expected, "expected source revision") + actual = command_output("git", "--no-pager", "rev-parse", "HEAD", cwd=directory) + if actual != expected: + raise SnapshotError(f"Unexpected source revision in {directory}: {actual}") + return actual + + +def helper_revision(expected: str) -> str: + require_revision(expected, "workflow revision") + actual = Path(__file__).with_name(HELPER_REVISION).read_text(encoding="utf-8").strip() + if actual != expected: + raise SnapshotError("Measurement helper is not from the tested workflow revision") + return actual + + +def helper_blob(directory: Path, revision: str, filename: str) -> bytes: + try: + content = subprocess.check_output( + ["git", "--no-pager", "-C", str(directory), "show", f"{revision}:scripts/{filename}"], + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as error: + details = error.stderr.decode("utf-8", errors="replace").strip() + raise SnapshotError(f"Missing helper {filename} at {revision}: {details}") from error + if not content: + raise SnapshotError(f"Empty helper {filename} at {revision}") + return content + + +def run_stage(args: argparse.Namespace) -> int: + source = args.source_dir.resolve() + destination = args.output_dir.resolve() + revision = source_revision(source, args.revision) + if destination.is_relative_to(source): + raise SnapshotError("Measurement helpers must be staged outside the discovery workspace") + files = {name: helper_blob(source, revision, name) for name in HELPER_FILES} + destination.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (destination / name).write_bytes(content) + (destination / HELPER_REVISION).write_text(revision + "\n", encoding="utf-8") + print(f"Staged measurement helpers from {revision}") + return 0 + + +def venv_python(prefix: Path) -> Path: + return prefix / "Scripts" / "python.exe" if os.name == "nt" else prefix / "bin" / "python" + + +def capture_context(args: argparse.Namespace) -> tuple[str, dict[str, Any]]: + if not args.target.strip(): + raise SnapshotError("Measurement target must not be empty") + if args.workload == "coverage" and args.pipenv_prefix is None: + raise SnapshotError("Coverage capture requires the created Pipenv prefix") + source = args.source_dir.resolve() + workspace = args.workspace.resolve() + revision = source_revision(source, args.expected_source) + helper = helper_revision(args.workflow_revision) + runner = {key: os.environ.get(key, "") for key in RUNNER_FIELDS} + if not all(value.strip() for value in runner.values()): + raise SnapshotError("Runner image context must not be empty") + tools = dict(COMMON_TOOLS) + if args.workload == "coverage": + tools.update(COVERAGE_TOOLS) + versions = { + name: command_output(*command, cwd=source if name in ("rustc", "cargo", "cargo-llvm-cov") else workspace) + for name, command in tools.items() + } + interpreters = { + "venv": command_output(str(venv_python(workspace / ".venv")), "--version", cwd=workspace), + "conda-base": command_output("conda", "run", "--name", "base", "python", "--version", cwd=workspace), + "conda-named": command_output( + "conda", "run", "--name", + "test-env1" if args.workload == "coverage" else "perf-test-env", + "python", "--version", cwd=workspace, + ), + } + context: dict[str, Any] = { + "runner": runner, + "installed_target": args.target, + "features": "ci" if args.workload == "coverage" else "ci-perf", + "helper_revision": helper, + "versions": versions, + "interpreters": interpreters, + } + if args.workload == "coverage": + for name, prefix in ( + ("venv2", workspace / ".venv2"), + ("pipenv", args.pipenv_prefix.resolve()), + ): + interpreters[name] = command_output(str(venv_python(prefix)), "--version", cwd=workspace) + for name in ("default", "dev"): + prefix = workspace / ".pixi" / "envs" / name + executable = prefix / "python.exe" if os.name == "nt" else prefix / "bin" / "python" + interpreters[f"pixi-{name}"] = command_output(str(executable), "--version", cwd=workspace) + interpreters["conda-prefix"] = command_output( + "conda", "run", "--prefix", str(workspace / "prefix-envs" / ".conda1"), + "python", "--version", cwd=workspace, + ) + interpreters["pyenv"] = "\n".join(sorted( + command_output("pyenv", "versions", "--bare", cwd=workspace).splitlines() + )) + context["coverage_target"] = "host" + else: + context["discovery_workspace"] = str(workspace) + return revision, context + + +def run_capture(args: argparse.Namespace) -> int: + revision, context = capture_context(args) + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "source-revision.txt").write_text(revision + "\n", encoding="utf-8") + (args.output_dir / "measurement-context.json").write_text( + json.dumps(context, indent=2, sort_keys=True) + "\n", encoding="utf-8", + ) + return 0 + + +def read_context(directory: Path, args: argparse.Namespace) -> dict[str, Any]: + path = directory / "measurement-context.json" + value = load_json(path) + required = { + "runner": RUNNER_FIELDS, + "versions": (*COMMON_TOOLS, *COVERAGE_TOOLS) if args.workload == "coverage" else tuple(COMMON_TOOLS), + "interpreters": ( + (*COMMON_INTERPRETERS, *COVERAGE_INTERPRETERS) + if args.workload == "coverage" else COMMON_INTERPRETERS + ), + } + for group, keys in required.items(): + values = value.get(group) + if not isinstance(values, dict) or any( + not isinstance(values.get(key), str) or not values[key].strip() for key in keys + ): + raise SnapshotError(f"Missing {group} measurement context in {path}") + if ( + value.get("features") != ("ci" if args.workload == "coverage" else "ci-perf") + or value.get("installed_target") != args.target + or value.get("helper_revision") != args.expected_head + ): + raise SnapshotError(f"Incompatible {args.workload} workload or helper revision in {path}") + if args.workload == "coverage": + if value.get("coverage_target") != "host": + raise SnapshotError(f"Incompatible coverage target in {path}") + elif args.workspace is None or value.get("discovery_workspace") != str(args.workspace.resolve()): + raise SnapshotError(f"Incompatible performance discovery workspace in {path}") + return value + + +def run_compare(args: argparse.Namespace) -> int: + title = "Test Coverage Report" if args.workload == "coverage" else "Performance Report" + try: + if not args.target.strip(): + raise SnapshotError("Measurement target must not be empty") + helper_revision(args.expected_head) + if args.workload == "coverage": + if args.measurements_result != "success": + raise SnapshotError("One or more coverage measurements or required proofs failed") + else: + for name in ( + "head_benchmark_outcome", "base_benchmark_outcome", + "head_metrics_outcome", "base_metrics_outcome", + ): + if getattr(args, name) != "success": + raise SnapshotError(f"Required benchmark or JSON extraction failed: {name}") + for directory, expected in ( + (args.current_dir, args.expected_head), (args.baseline_dir, args.expected_base), + ): + require_revision(expected, "expected source revision") + path = directory / "source-revision.txt" + if path.read_text(encoding="utf-8").strip() != expected: + raise SnapshotError(f"Unexpected source revision in {path}") + current = read_context(args.current_dir, args) + baseline = read_context(args.baseline_dir, args) + if current != baseline: + raise SnapshotError( + f"{args.workload.capitalize()} runner, tool, or fixture versions differ; inspect both measurement contexts" + ) + except (OSError, UnicodeError, SnapshotError) as error: + print(f"Invalid {args.workload} measurement: {error}", file=sys.stderr) + write_report( + f"## {title} ({args.platform})\n\n:x: **Invalid snapshot:** {error}\n", + args.report, args.summary, + ) + return 1 + filename = "lcov.info" if args.workload == "coverage" else "metrics.json" + comparator = run_coverage if args.workload == "coverage" else run_performance + return comparator(argparse.Namespace( + current=args.current_dir / filename, baseline=args.baseline_dir / filename, + platform=args.platform, report=args.report, summary=args.summary, + )) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + stage = commands.add_parser("stage", help="Export the tested workflow's helpers before checking out old source") + stage.add_argument("--source-dir", type=Path, required=True) + stage.add_argument("--output-dir", type=Path, required=True) + stage.add_argument("--revision", required=True) + stage.set_defaults(handler=run_stage) + capture = commands.add_parser("capture", help="Record a revision's tool and fixture context") + capture.add_argument("workload", choices=("coverage", "performance")) + capture.add_argument("--source-dir", type=Path, required=True) + capture.add_argument("--workspace", type=Path, required=True) + capture.add_argument("--output-dir", type=Path, required=True) + capture.add_argument("--target", required=True) + capture.add_argument("--expected-source", required=True) + capture.add_argument("--workflow-revision", required=True) + capture.add_argument("--pipenv-prefix", type=Path) + capture.set_defaults(handler=run_capture) + compare = commands.add_parser("compare", help="Validate provenance and run the unchanged numerical gate") + compare.add_argument("workload", choices=("coverage", "performance")) + compare.add_argument("--current-dir", type=Path, required=True) + compare.add_argument("--baseline-dir", type=Path, required=True) + compare.add_argument("--expected-head", required=True) + compare.add_argument("--expected-base", required=True) + compare.add_argument("--target", required=True) + compare.add_argument("--workspace", type=Path) + compare.add_argument("--platform", required=True) + compare.add_argument("--report", type=Path, required=True) + compare.add_argument("--summary", type=Path) + compare.add_argument("--measurements-result") + for name in ("head-benchmark-outcome", "base-benchmark-outcome", "head-metrics-outcome", "base-metrics-outcome"): + compare.add_argument("--" + name) + compare.set_defaults(handler=run_compare) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return args.handler(args) + except (OSError, UnicodeError, SnapshotError) as error: + print(f"Invalid measurement helper operation: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_quality_workflows.py b/scripts/tests/test_quality_workflows.py index 4b524734..84f8a6dd 100644 --- a/scripts/tests/test_quality_workflows.py +++ b/scripts/tests/test_quality_workflows.py @@ -5,17 +5,23 @@ import json import os import re +import shutil import subprocess import sys import tempfile -import textwrap import unittest from pathlib import Path +from unittest.mock import patch from test_quality_snapshot import performance_snapshot, write_lcov ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) + +import quality_measurement as measurement # noqa: E402 +from quality_snapshot import SnapshotError # noqa: E402 + SAME_REPOSITORY_PR = ( "github.event.pull_request.head.repo.full_name == github.repository" ) @@ -47,9 +53,12 @@ def step_property(step: str, name: str) -> str | None: return None -def workflow_python(step: str) -> str: - source = step.split(" python - <<'PY'\n", 1)[1] - return textwrap.dedent(source.split("\n PY", 1)[0]) +def copy_measurement_helpers(directory: Path, revision: str) -> Path: + directory.mkdir() + for filename in ("quality_measurement.py", "quality_snapshot.py"): + shutil.copyfile(ROOT / "scripts" / filename, directory / filename) + (directory / "helper-revision.txt").write_text(revision + "\n", encoding="utf-8") + return directory / "quality_measurement.py" class QualityWorkflowTests(unittest.TestCase): @@ -80,8 +89,14 @@ def test_macos_runners_keep_both_native_architecture_workloads(self) -> None: ("macos-15-intel", "x86_64-apple-darwin"), ("macos-15", "aarch64-apple-darwin"), ], - "perf-tests.yml": [("macos-15-intel", "x86_64-apple-darwin")], - "perf-baseline.yml": [("macos-15-intel", "x86_64-apple-darwin")], + "perf-tests.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ], + "perf-baseline.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ], } for workflow, targets in expected.items(): with self.subTest(workflow=workflow): @@ -110,6 +125,24 @@ def test_coverage_pairs_clean_exact_revision_jobs_from_the_same_attempt(self) -> ) self.assertIn("needs: measurements", text) self.assertIn("MEASUREMENTS_RESULT: ${{ needs.measurements.result }}", text) + self.assertIn("ref: ${{ github.sha }}", steps["Checkout Workflow"]) + self.assertLess( + text.index("- name: Stage Coverage Measurement Helper"), + text.index("- name: Checkout Measurement Source"), + ) + self.assertIn( + '--output-dir "$RUNNER_TEMP/quality-tools"', steps["Stage Coverage Measurement Helper"], + ) + self.assertIn( + '--revision "$GITHUB_SHA"', steps["Stage Coverage Measurement Helper"], + ) + self.assertIn( + '"$RUNNER_TEMP/quality-tools/quality_measurement.py" capture coverage', + steps["Capture Coverage Measurement Context"], + ) + self.assertIn('--expected-source "$MEASURED_SHA"', steps["Capture Coverage Measurement Context"]) + self.assertIn('--workflow-revision "$GITHUB_SHA"', steps["Capture Coverage Measurement Context"]) + self.assertNotIn("python - <<", text) self.assertIn('prefix="$(pipenv --venv)"', steps["Create a Pipenv Environment"]) self.assertLess( text.index("- name: Create a Pipenv Environment"), @@ -152,6 +185,12 @@ def test_performance_builds_each_harness_before_timing_a_shared_workspace(self) self.assertNotIn("GITHUB_WORKSPACE:", text) self.assertNotIn("wait_for_baseline.py", text) self.assertNotIn("dawidd6/action-download-artifact", text) + self.assertIn("comment_header: perf-macos-arm64", text) + self.assertIn("comment_header: perf-macos\n", text) + self.assertIn("capture performance", steps["Capture Performance Measurement Context"]) + self.assertIn('--source-dir "$RUNNER_TEMP/perf-base"', steps["Capture Performance Measurement Context"]) + self.assertIn('--expected-source "$BASE_SHA"', steps["Capture Performance Measurement Context"]) + self.assertNotIn("python - <<", text) for step in ("Extract Performance Metrics", "Extract Exact Base Performance Metrics"): self.assertIn("always()", step_property(steps[step], "if")) self.assertIn("python -m json.tool", steps[step]) @@ -252,11 +291,11 @@ def test_report_artifacts_are_uploaded_after_comparison(self) -> None: class MeasurementComparisonTests(unittest.TestCase): def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None): baseline_name = "baseline-coverage" if kind == "coverage" else "baseline-perf" - workflow = "coverage.yml" if kind == "coverage" else "perf-tests.yml" - step_name = "Compare Coverage Snapshot" if kind == "coverage" else "Compare Performance Snapshot" - code = workflow_python(workflow_steps(".github/workflows/" + workflow)[step_name]) with tempfile.TemporaryDirectory() as directory: - root = Path(directory) + parent = Path(directory) + root = parent / "workspace" + root.mkdir() + helper = copy_measurement_helpers(parent / "tools", "a" * 40) baseline = root / baseline_name baseline.mkdir() context = { @@ -265,6 +304,7 @@ def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None ), "installed_target": "x86_64-unknown-linux-musl", "features": "ci" if kind == "coverage" else "ci-perf", + "helper_revision": "a" * 40, "versions": dict.fromkeys(("rustc", "cargo", "python", "conda"), "tool-version"), "interpreters": dict.fromkeys( ("conda-base", "conda-named", "venv"), "Python 3.12.1" @@ -304,7 +344,14 @@ def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None del base_context["versions"]["rustc"] elif invalid == "workload": current_context["features"] = base_context["features"] = "other-features" - elif invalid not in (None, "missing", "malformed", "revision"): + elif invalid == "helper": + base_context["helper_revision"] = "b" * 40 + elif invalid == "coverage-only": + del current_context["versions"]["pixi"] + del base_context["versions"]["pixi"] + elif invalid not in ( + None, "missing", "malformed", "revision", "missing-helper", "missing-dependency", + ): raise ValueError(f"Unknown invalid measurement case: {invalid}") (root / "measurement-context.json").write_text(json.dumps(current_context), encoding="utf-8") (baseline / "measurement-context.json").write_text(json.dumps(base_context), encoding="utf-8") @@ -316,16 +363,16 @@ def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None (baseline / "measurement-context.json").unlink() elif invalid == "malformed": (baseline / "measurement-context.json").write_text("{", encoding="utf-8") + elif invalid == "missing-helper": + (helper.parent / "helper-revision.txt").unlink() + elif invalid == "missing-dependency": + (helper.parent / "quality_snapshot.py").unlink() environment = { **os.environ, "PYTHONPATH": str(ROOT / "scripts"), "PYTHONDONTWRITEBYTECODE": "1", - "GITHUB_WORKSPACE": str(root), - "GITHUB_STEP_SUMMARY": str(root / "summary.md"), - "PLATFORM": "Linux", - "MEASUREMENT_TARGET": "x86_64-unknown-linux-musl", - "HEAD_SHA": "a" * 40, - "BASE_SHA": "b" * 40, + } + results = { "MEASUREMENTS_RESULT": "success", **dict.fromkeys(( "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", @@ -333,13 +380,32 @@ def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None ), "success"), **(outcomes or {}), } - result = subprocess.run( - [sys.executable, "-B", "-c", code], cwd=root, env=environment, - capture_output=True, text=True, check=False, - ) report_path = root / ( "coverage-report.md" if kind == "coverage" else "performance-report.md" ) + arguments = [ + sys.executable, "-B", str(helper), "compare", kind, + "--current-dir", str(root), "--baseline-dir", str(baseline), + "--expected-head", "a" * 40, "--expected-base", "b" * 40, + "--target", "x86_64-unknown-linux-musl", "--workspace", str(root), + "--platform", "Linux", "--report", str(report_path), + "--summary", str(root / "summary.md"), + ] + for variable, flag in ( + ("MEASUREMENTS_RESULT", "--measurements-result"), + ("HEAD_BENCHMARK_OUTCOME", "--head-benchmark-outcome"), + ("BASE_BENCHMARK_OUTCOME", "--base-benchmark-outcome"), + ("HEAD_METRICS_OUTCOME", "--head-metrics-outcome"), + ("BASE_METRICS_OUTCOME", "--base-metrics-outcome"), + ): + arguments.extend((flag, results[variable])) + result = subprocess.run( + arguments, cwd=root, env=environment, + capture_output=True, text=True, check=False, + ) + if invalid == "missing-dependency": + self.assertFalse(report_path.exists()) + return result, "" self.assertTrue(report_path.is_file(), result.stderr) return result, report_path.read_text(encoding="utf-8") @@ -358,7 +424,7 @@ def test_invalid_provenance_or_context_cannot_pass_unchanged_snapshots(self) -> for kind in ("coverage", "performance"): for invalid in ( "missing", "malformed", "incomplete", "tool", - "image", "fixture", "workload", "revision", + "image", "fixture", "workload", "revision", "helper", "missing-helper", ): with self.subTest(kind=kind, invalid=invalid): result, report = self.run_measurement(kind, invalid=invalid) @@ -378,6 +444,131 @@ def test_failed_execution_or_extraction_cannot_pass_complete_snapshots(self) -> self.assertIn("Invalid snapshot", report) self.assertNotIn("Within regression budget", report) + def test_missing_staged_dependency_cannot_fall_back_to_pythonpath(self) -> None: + result, _ = self.run_measurement("coverage", invalid="missing-dependency") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Missing measurement helper dependency", result.stderr) + + def test_coverage_still_requires_its_manager_specific_context(self) -> None: + result, report = self.run_measurement("coverage", invalid="coverage-only") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Missing versions measurement context", report) + + +class MeasurementCaptureTests(unittest.TestCase): + def test_capture_keeps_base_source_separate_from_helper_and_discovery_workspace(self) -> None: + for workload in ("coverage", "performance"): + with self.subTest(workload=workload), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "old-base" + workspace = root / "workspace" + source.mkdir() + workspace.mkdir() + helper = copy_measurement_helpers(root / "tools", "a" * 40) + output = workspace / "baseline" + arguments = [ + "capture", workload, "--source-dir", str(source), + "--workspace", str(workspace), "--output-dir", str(output), + "--target", "aarch64-apple-darwin", "--expected-source", "b" * 40, + "--workflow-revision", "a" * 40, + ] + if workload == "coverage": + arguments.extend(("--pipenv-prefix", str(workspace / "created-pipenv"))) + calls = [] + + def command(*args, cwd): + calls.append((args, cwd)) + return "b" * 40 if args[0] == "git" else "version" + + with ( + patch.object(measurement, "__file__", str(helper)), + patch.object(measurement, "command_output", side_effect=command), + patch.dict(os.environ, dict.fromkeys(measurement.RUNNER_FIELDS, "runner")), + ): + self.assertEqual(measurement.main(arguments), 0) + context = json.loads((output / "measurement-context.json").read_text(encoding="utf-8")) + self.assertEqual((output / "source-revision.txt").read_text().strip(), "b" * 40) + self.assertEqual(context["helper_revision"], "a" * 40) + self.assertFalse((source / "scripts" / "quality_measurement.py").exists()) + for args, cwd in calls: + self.assertEqual( + cwd, source.resolve() if args[0] in ("git", "rustc", "cargo") else workspace.resolve() + ) + if workload == "coverage": + self.assertIn("pipenv", context["versions"]) + self.assertIn("pixi-dev", context["interpreters"]) + self.assertIn("coverage_target", context) + self.assertTrue(any("test-env1" in args for args, _ in calls)) + else: + self.assertNotIn("pipenv", context["versions"]) + self.assertNotIn("pixi-dev", context["interpreters"]) + self.assertEqual(context["discovery_workspace"], str(workspace.resolve())) + self.assertTrue(any("perf-test-env" in args for args, _ in calls)) + + def test_capture_rejects_head_reused_as_base_before_writing_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / "evidence" + arguments = [ + "capture", "performance", "--source-dir", str(root), + "--workspace", str(root), "--output-dir", str(output), + "--target", "aarch64-apple-darwin", "--expected-source", "b" * 40, + "--workflow-revision", "a" * 40, + ] + with patch.object(measurement, "command_output", return_value="a" * 40): + self.assertEqual(measurement.main(arguments), 1) + self.assertFalse(output.exists()) + + def test_stage_exports_immutable_helpers_without_copying_measured_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "workflow" + source.mkdir() + destination = root / "tools" + files = { + name: (ROOT / "scripts" / name).read_bytes() + for name in ("quality_measurement.py", "quality_snapshot.py") + } + arguments = [ + "stage", "--source-dir", str(source), "--output-dir", str(destination), + "--revision", "a" * 40, + ] + with ( + patch.object(measurement, "command_output", return_value="a" * 40), + patch.object( + measurement, "helper_blob", + side_effect=lambda path, revision, filename: files[filename], + ) as blobs, + ): + self.assertEqual(measurement.main(arguments), 0) + for name, content in files.items(): + self.assertEqual((destination / name).read_bytes(), content) + self.assertEqual((destination / "helper-revision.txt").read_text().strip(), "a" * 40) + self.assertEqual(list(source.iterdir()), []) + self.assertEqual(blobs.call_count, 2) + self.assertTrue(all(call.args[1] == "a" * 40 for call in blobs.call_args_list)) + + def test_stage_rejects_missing_helpers_and_workspace_injection(self) -> None: + for invalid in ("revision", "missing-helper", "inside-workspace"): + with self.subTest(invalid=invalid), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "workflow" + source.mkdir() + destination = source / "tools" if invalid == "inside-workspace" else root / "tools" + arguments = [ + "stage", "--source-dir", str(source), "--output-dir", str(destination), + "--revision", "a" * 40, + ] + with ( + patch.object( + measurement, "command_output", + return_value=("b" if invalid == "revision" else "a") * 40, + ), + patch.object(measurement, "helper_blob", side_effect=SnapshotError("Missing helper")), + ): + self.assertEqual(measurement.main(arguments), 1) + self.assertFalse((destination / "helper-revision.txt").exists()) + if __name__ == "__main__": unittest.main()