diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5f426a7..90ff4970 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,9 +26,9 @@ jobs: # target: aarch64-unknown-linux-gnu # - os: ubuntu-latest # target: arm-unknown-linux-gnueabihf - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin - - os: macos-14 + - os: macos-15 target: aarch64-apple-darwin # - os: ubuntu-latest # target: x86_64-unknown-linux-gnu diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index a17f7225..c8c9dcc4 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -46,10 +46,14 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Linux) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Check Conda version run: conda info --all diff --git a/.github/workflows/coverage-macos.yml b/.github/workflows/coverage-macos.yml index c7586a5c..a36763d8 100644 --- a/.github/workflows/coverage-macos.yml +++ b/.github/workflows/coverage-macos.yml @@ -12,7 +12,7 @@ permissions: jobs: coverage: - runs-on: macos-14 + runs-on: macos-15 timeout-minutes: 35 steps: - name: Checkout diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 43d3a62f..a5acb01a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,12 +14,14 @@ permissions: pull-requests: write jobs: - coverage: - name: Coverage + measurements: + name: Coverage measurement (${{ matrix.platform }}, ${{ matrix.revision }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, windows-latest] + revision: [base, head] include: - os: ubuntu-latest target: x86_64-unknown-linux-musl @@ -30,14 +32,17 @@ jobs: platform: Windows comment_header: coverage-windows steps: - - name: Checkout + - name: Checkout Workflow uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ github.sha }} fetch-depth: 0 persist-credentials: false - name: Post Coverage Started Comment - if: github.event.pull_request.head.repo.full_name == github.repository + if: >- + matrix.revision == 'head' && + github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: @@ -58,16 +63,35 @@ jobs: run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v shell: bash + - name: Stage Coverage Measurement Helper + run: >- + python scripts/quality_measurement.py stage + --source-dir "$GITHUB_WORKSPACE" + --output-dir "$RUNNER_TEMP/quality-tools" + --revision "$GITHUB_SHA" + shell: bash + + - name: Checkout Measurement Source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Add Conda to PATH (Windows) if: startsWith(matrix.os, 'windows') run: | $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Linux) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Check Conda version run: conda info --all @@ -86,7 +110,12 @@ jobs: run: pipenv --version - name: Create a Pipenv Environment - run: pipenv install + id: pipenv + run: | + pipenv install + prefix="$(pipenv --venv)" + echo "prefix=$prefix" >> "$GITHUB_OUTPUT" + shell: bash - name: Install virtualenvwrapper (Linux) if: startsWith(matrix.os, 'ubuntu') @@ -197,6 +226,22 @@ jobs: run: cargo fetch shell: bash + - name: Capture Coverage Measurement Context + env: + MEASUREMENT_TARGET: ${{ matrix.target }} + MEASURED_SHA: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }} + CREATED_PIPENV_PREFIX: ${{ steps.pipenv.outputs.prefix }} + run: >- + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" capture coverage + --source-dir "$GITHUB_WORKSPACE" + --workspace "$GITHUB_WORKSPACE" + --output-dir "$GITHUB_WORKSPACE" + --target "$MEASUREMENT_TARGET" + --expected-source "$MEASURED_SHA" + --workflow-revision "$GITHUB_SHA" + --pipenv-prefix "$CREATED_PIPENV_PREFIX" + shell: bash + - name: Run Tests with Coverage id: coverage run: cargo llvm-cov --features ci --lcov --output-path lcov.info -- --nocapture --test-threads=1 @@ -220,40 +265,85 @@ jobs: --output production-coverage shell: bash - - name: Wait for Exact PR Base Coverage - id: wait_for_base_coverage + - name: Upload Coverage Measurement if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-measurement-${{ matrix.os }}-${{ matrix.revision }}-${{ github.run_attempt }} + path: | + lcov.info + measurement-context.json + source-revision.txt + production-coverage/ + subprocess-coverage/ + subprocess-coverage.json + if-no-files-found: error + + coverage: + name: Coverage + needs: measurements + if: always() + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + platform: Linux + comment_header: coverage-linux + - os: windows-latest + target: x86_64-pc-windows-msvc + platform: Windows + comment_header: coverage-windows + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set Python to PATH + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Stage Coverage Comparison Helper run: >- - python scripts/wait_for_baseline.py - --repository "${{ github.repository }}" - --workflow coverage-baseline.yml - --commit "${{ github.event.pull_request.base.sha }}" - --artifact "coverage-baseline-${{ matrix.os }}" - --timeout-seconds 1200 - --poll-seconds 20 - env: - GITHUB_TOKEN: ${{ github.token }} + python scripts/quality_measurement.py stage + --source-dir "$GITHUB_WORKSPACE" + --output-dir "$RUNNER_TEMP/quality-tools" + --revision "$GITHUB_SHA" shell: bash + - name: Download Head Coverage Measurement + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: coverage-measurement-${{ matrix.os }}-head-${{ github.run_attempt }} + - name: Download Exact PR Base Coverage - if: always() && steps.wait_for_base_coverage.outcome == 'success' - uses: dawidd6/action-download-artifact@eab87c9830c39eff17e5a6eadb20bfb4bc880477 # v27 + if: always() + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - workflow: coverage-baseline.yml - commit: ${{ github.event.pull_request.base.sha }} - workflow_conclusion: success - name: coverage-baseline-${{ matrix.os }} + name: coverage-measurement-${{ matrix.os }}-base-${{ github.run_attempt }} path: baseline-coverage - check_artifacts: true - search_artifacts: true - name: Compare Coverage Snapshot if: always() + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.sha }} + MEASUREMENT_TARGET: ${{ matrix.target }} + MEASUREMENTS_RESULT: ${{ needs.measurements.result }} + PLATFORM: ${{ matrix.platform }} run: >- - python scripts/quality_snapshot.py coverage - --current lcov.info - --baseline baseline-coverage/lcov.info - --platform "${{ matrix.platform }}" + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" compare coverage + --current-dir . + --baseline-dir baseline-coverage + --expected-head "$HEAD_SHA" + --expected-base "$BASE_SHA" + --target "$MEASUREMENT_TARGET" + --measurements-result "$MEASUREMENTS_RESULT" + --platform "$PLATFORM" --report coverage-report.md --summary "$GITHUB_STEP_SUMMARY" shell: bash @@ -265,6 +355,9 @@ jobs: name: coverage-pr-${{ matrix.os }} path: | lcov.info + measurement-context.json + source-revision.txt + baseline-coverage/ coverage-report.md production-coverage/ subprocess-coverage/ diff --git a/.github/workflows/perf-baseline.yml b/.github/workflows/perf-baseline.yml index 23821061..b91bf870 100644 --- a/.github/workflows/perf-baseline.yml +++ b/.github/workflows/perf-baseline.yml @@ -24,8 +24,12 @@ jobs: target: x86_64-unknown-linux-musl - os: windows-latest target: x86_64-pc-windows-msvc - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin + conda_arch: x86_64 + - os: macos-15 + target: aarch64-apple-darwin + conda_arch: arm64 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -45,20 +49,34 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Linux) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Install Conda + add to PATH (macOS) if: startsWith(matrix.os, 'macos') + env: + CONDA_ARCH: ${{ matrix.conda_arch }} run: | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - bash ~/miniconda.sh -b -p ~/miniconda - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV + if [[ "$(uname -m)" != "$CONDA_ARCH" ]]; then + echo "Conda installer architecture $CONDA_ARCH does not match runner $(uname -m)" >&2 + exit 1 + fi + curl --fail --location --show-error --output "$RUNNER_TEMP/miniconda.sh" \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh" + bash "$RUNNER_TEMP/miniconda.sh" -b -p "$HOME/miniconda" + echo "PATH=$PATH:$HOME/miniconda/bin" >> "$GITHUB_ENV" + echo "CONDA=$HOME/miniconda" >> "$GITHUB_ENV" shell: bash + - name: Check Conda version + run: conda info --all + - name: Create test Conda environment run: conda create -n perf-test-env python=3.12 -y diff --git a/.github/workflows/perf-tests.yml b/.github/workflows/perf-tests.yml index 0ccdf8d4..ec0cccbe 100644 --- a/.github/workflows/perf-tests.yml +++ b/.github/workflows/perf-tests.yml @@ -30,13 +30,22 @@ jobs: target: x86_64-unknown-linux-musl platform: Linux comment_header: perf-linux - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin + conda_arch: x86_64 platform: macOS comment_header: perf-macos + - os: macos-15 + target: aarch64-apple-darwin + conda_arch: arm64 + platform: macOS ARM64 + comment_header: perf-macos-arm64 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false - name: Post In-Progress Comment if: >- @@ -49,7 +58,7 @@ jobs: message: | ## Performance Report (${{ matrix.platform }}) :hourglass_flowing_sand: - Running performance tests against baseline `${{ github.event.pull_request.base.sha }}`. + Measuring this revision and exact base `${{ github.event.pull_request.base.sha }}` on the same runner. - name: Set Python to PATH uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -60,26 +69,48 @@ jobs: run: python -m unittest discover -s scripts/tests -p 'test_*.py' -v shell: bash + - name: Stage Performance Measurement Helper + run: >- + python scripts/quality_measurement.py stage + --source-dir "$GITHUB_WORKSPACE" + --output-dir "$RUNNER_TEMP/quality-tools" + --revision "$GITHUB_SHA" + shell: bash + - name: Add Conda to PATH (Windows) if: startsWith(matrix.os, 'windows') run: | $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH (Ubuntu) + - name: Set up Conda (Linux) if: startsWith(matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Install Conda + add to PATH (macOS) if: startsWith(matrix.os, 'macos') + env: + CONDA_ARCH: ${{ matrix.conda_arch }} run: | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - bash ~/miniconda.sh -b -p ~/miniconda - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV + if [[ "$(uname -m)" != "$CONDA_ARCH" ]]; then + echo "Conda installer architecture $CONDA_ARCH does not match runner $(uname -m)" >&2 + exit 1 + fi + curl --fail --location --show-error --output "$RUNNER_TEMP/miniconda.sh" \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh" + bash "$RUNNER_TEMP/miniconda.sh" -b -p "$HOME/miniconda" + echo "PATH=$PATH:$HOME/miniconda/bin" >> "$GITHUB_ENV" + echo "CONDA=$HOME/miniconda" >> "$GITHUB_ENV" shell: bash + - name: Check Conda version + run: conda info --all + - name: Create test Conda environment run: conda create -n perf-test-env python=3.12 -y @@ -93,26 +124,98 @@ jobs: toolchain: stable targets: ${{ matrix.target }} + - name: Prepare Exact Base Worktree + id: baseline + env: + PR_BASE: ${{ github.event.pull_request.base.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + base="$PR_BASE" + else + base="$(git rev-parse --verify "refs/remotes/origin/$DEFAULT_BRANCH^{commit}")" + fi + if [[ ! "$base" =~ ^[0-9a-f]{40}$ ]]; then + echo "Missing or invalid exact base revision" >&2 + exit 1 + fi + git worktree add --detach "$RUNNER_TEMP/perf-base" "$base" + mkdir -p baseline-perf + echo "sha=$base" >> "$GITHUB_OUTPUT" + shell: bash + + - name: Capture Performance Measurement Context + id: measurement_context + env: + BASE_SHA: ${{ steps.baseline.outputs.sha }} + HEAD_SHA: ${{ github.sha }} + MEASUREMENT_TARGET: ${{ matrix.target }} + run: | + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" capture performance \ + --source-dir "$GITHUB_WORKSPACE" --workspace "$GITHUB_WORKSPACE" \ + --output-dir "$GITHUB_WORKSPACE" --target "$MEASUREMENT_TARGET" \ + --expected-source "$HEAD_SHA" --workflow-revision "$GITHUB_SHA" + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" capture performance \ + --source-dir "$RUNNER_TEMP/perf-base" --workspace "$GITHUB_WORKSPACE" \ + --output-dir "$GITHUB_WORKSPACE/baseline-perf" --target "$MEASUREMENT_TARGET" \ + --expected-source "$BASE_SHA" --workflow-revision "$GITHUB_SHA" + shell: bash + - name: Cargo Fetch - run: cargo fetch + run: | + cargo fetch --locked + cd "$RUNNER_TEMP/perf-base" + cargo fetch --locked shell: bash - - name: Build Release - run: cargo build --release --target ${{ matrix.target }} + - name: Build Release and Benchmark Harnesses + id: build + env: + MEASUREMENT_TARGET: ${{ matrix.target }} + run: | + for source in "$RUNNER_TEMP/perf-base" "$GITHUB_WORKSPACE"; do + ( + cd "$source" + cargo build --frozen --release --target-dir target --target "$MEASUREMENT_TARGET" + cargo test --frozen --release --target-dir target --features ci-perf --target "$MEASUREMENT_TARGET" \ + --test e2e_performance --no-run + git diff --exit-code -- Cargo.toml Cargo.lock crates + ) + done + shell: bash + + - name: Run Exact Base Performance Tests + id: base_benchmark + env: + MEASUREMENT_TARGET: ${{ matrix.target }} + RUST_BACKTRACE: 1 + RUST_LOG: warn + run: | + set -o pipefail + cd "$RUNNER_TEMP/perf-base" + cargo test --frozen --release --target-dir target --features ci-perf --target "$MEASUREMENT_TARGET" \ + --test e2e_performance test_performance_summary -- --nocapture 2>&1 \ + | tee "$GITHUB_WORKSPACE/baseline-perf/perf-output.txt" shell: bash - name: Run Performance Tests id: benchmark + if: >- + always() && steps.build.outcome == 'success' && + steps.measurement_context.outcome == 'success' run: | set -o pipefail - cargo test --release --features ci-perf --target ${{ matrix.target }} --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt + cargo test --frozen --release --target-dir target --features ci-perf --target "$MEASUREMENT_TARGET" \ + --test e2e_performance test_performance_summary -- --nocapture 2>&1 | tee perf-output.txt env: + MEASUREMENT_TARGET: ${{ matrix.target }} RUST_BACKTRACE: 1 RUST_LOG: warn shell: bash - name: Extract Performance Metrics - if: steps.benchmark.outcome == 'success' + id: metrics + if: always() && steps.benchmark.outcome == 'success' run: | if ! grep -q "JSON metrics:" perf-output.txt; then echo "Performance test produced no JSON metrics" >&2 @@ -123,54 +226,43 @@ jobs: cat metrics.json shell: bash - - name: Wait for Exact PR Base Performance - id: wait_for_base_performance - if: always() && github.event_name == 'pull_request' - run: >- - python scripts/wait_for_baseline.py - --repository "${{ github.repository }}" - --workflow perf-baseline.yml - --commit "${{ github.event.pull_request.base.sha }}" - --artifact "perf-baseline-${{ matrix.os }}" - --timeout-seconds 1200 - --poll-seconds 20 - env: - GITHUB_TOKEN: ${{ github.token }} + - name: Extract Exact Base Performance Metrics + id: base_metrics + if: always() && steps.base_benchmark.outcome == 'success' + run: | + if ! grep -q "JSON metrics:" baseline-perf/perf-output.txt; then + echo "Exact base performance test produced no JSON metrics" >&2 + exit 1 + fi + sed -n '/JSON metrics:/,/^}/p' baseline-perf/perf-output.txt | tail -n +2 > baseline-perf/metrics.json + python -m json.tool baseline-perf/metrics.json > /dev/null + cat baseline-perf/metrics.json shell: bash - - name: Download Exact PR Base Performance - if: >- - always() && github.event_name == 'pull_request' && - steps.wait_for_base_performance.outcome == 'success' - uses: dawidd6/action-download-artifact@eab87c9830c39eff17e5a6eadb20bfb4bc880477 # v27 - with: - workflow: perf-baseline.yml - commit: ${{ github.event.pull_request.base.sha }} - workflow_conclusion: success - name: perf-baseline-${{ matrix.os }} - path: baseline-perf - check_artifacts: true - search_artifacts: true - - - name: Download Main Performance for Manual Run - if: always() && github.event_name == 'workflow_dispatch' - uses: dawidd6/action-download-artifact@eab87c9830c39eff17e5a6eadb20bfb4bc880477 # v27 - with: - workflow: perf-baseline.yml - branch: main - workflow_conclusion: success - name: perf-baseline-${{ matrix.os }} - path: baseline-perf - check_artifacts: true - search_artifacts: true - - name: Compare Performance Snapshot if: always() + env: + BASE_SHA: ${{ steps.baseline.outputs.sha }} + HEAD_SHA: ${{ github.sha }} + MEASUREMENT_TARGET: ${{ matrix.target }} + HEAD_BENCHMARK_OUTCOME: ${{ steps.benchmark.outcome }} + BASE_BENCHMARK_OUTCOME: ${{ steps.base_benchmark.outcome }} + HEAD_METRICS_OUTCOME: ${{ steps.metrics.outcome }} + BASE_METRICS_OUTCOME: ${{ steps.base_metrics.outcome }} + PLATFORM: ${{ matrix.platform }} run: >- - python scripts/quality_snapshot.py performance - --current metrics.json - --baseline baseline-perf/metrics.json - --platform "${{ matrix.platform }}" + python "$RUNNER_TEMP/quality-tools/quality_measurement.py" compare performance + --current-dir . + --baseline-dir baseline-perf + --expected-head "$HEAD_SHA" + --expected-base "$BASE_SHA" + --target "$MEASUREMENT_TARGET" + --workspace "$GITHUB_WORKSPACE" + --head-benchmark-outcome "$HEAD_BENCHMARK_OUTCOME" + --base-benchmark-outcome "$BASE_BENCHMARK_OUTCOME" + --head-metrics-outcome "$HEAD_METRICS_OUTCOME" + --base-metrics-outcome "$BASE_METRICS_OUTCOME" + --platform "$PLATFORM" --report performance-report.md --summary "$GITHUB_STEP_SUMMARY" shell: bash @@ -182,6 +274,10 @@ jobs: name: perf-pr-${{ matrix.os }} path: | metrics.json + measurement-context.json + source-revision.txt + baseline-perf/ + perf-output.txt performance-report.md if-no-files-found: ignore diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index a916d2e4..ef421cc3 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -32,11 +32,13 @@ jobs: - os: ubuntu-latest target: x86_64-unknown-linux-musl run_cli: "yes" - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin + conda_arch: x86_64 run_cli: "yes" - - os: macos-14 + - os: macos-15 target: aarch64-apple-darwin + conda_arch: arm64 run_cli: "yes" steps: - name: Checkout @@ -58,18 +60,29 @@ jobs: $path = $env:PATH + ";" + $env:CONDA + "\condabin" echo "PATH=$path" >> $env:GITHUB_ENV - - name: Add Conda to PATH + - name: Set up Conda (Linux) if: startsWith( matrix.os, 'ubuntu') - run: echo "PATH=$PATH:$CONDA/condabin" >> $GITHUB_ENV - shell: bash + uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + activate-environment: "" + auto-activate: false + run-init: false + conda-remove-defaults: false - name: Install Conda + add to PATH if: startsWith( matrix.os, 'macos') + env: + CONDA_ARCH: ${{ matrix.conda_arch }} run: | - curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - bash ~/miniconda.sh -b -p ~/miniconda - echo "PATH=$PATH:$HOME/miniconda/bin" >> $GITHUB_ENV - echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV + if [[ "$(uname -m)" != "$CONDA_ARCH" ]]; then + echo "Conda installer architecture $CONDA_ARCH does not match runner $(uname -m)" >&2 + exit 1 + fi + curl --fail --location --show-error --output "$RUNNER_TEMP/miniconda.sh" \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh" + bash "$RUNNER_TEMP/miniconda.sh" -b -p "$HOME/miniconda" + echo "PATH=$PATH:$HOME/miniconda/bin" >> "$GITHUB_ENV" + echo "CONDA=$HOME/miniconda" >> "$GITHUB_ENV" shell: bash - name: Check Conda version @@ -459,10 +472,10 @@ jobs: # target: aarch64-unknown-linux-gnu # - os: ubuntu-latest # target: arm-unknown-linux-gnueabihf - - os: macos-latest + - os: macos-15-intel target: x86_64-apple-darwin run_cli: "yes" - - os: macos-14 + - os: macos-15 target: aarch64-apple-darwin run_cli: "yes" # - os: ubuntu-latest diff --git a/docs/QUALITY_SNAPSHOTS.md b/docs/QUALITY_SNAPSHOTS.md index df35bed8..86f6c435 100644 --- a/docs/QUALITY_SNAPSHOTS.md +++ b/docs/QUALITY_SNAPSHOTS.md @@ -1,11 +1,65 @@ # Quality snapshots -PET uses pull-request snapshots to prevent performance and coverage drift. Each pull request is compared with artifacts produced for the exact pull-request base commit, not the latest moving `main` tip. +PET uses pull-request snapshots to prevent performance and coverage drift. Each pull request +measures its exact base commit with the current workflow setup, rather than compare against a +moving `main` tip or a historical measurement from a different compiler or runner image. + +## Comparable measurements + +Linux and Windows coverage run the exact base and tested PR revision in separate fresh jobs. +Both use the same installed-manager `ci` workload and fixture setup. They must not be measured +sequentially against shared mutable fixtures: the Unix Conda CI tests create environments and +install Mamba into the base installation. Native ARM64 macOS coverage retains its separate +default-feature workload and same-runner base measurement. + +`scripts/quality_measurement.py` provides argument-driven staging, context capture, and +provenance validation for both workloads, delegating numerical gates to `quality_snapshot.py`. +Coverage still requires its additional llvm-cov/Pipenv/Pyenv/Pixi tools and fixture interpreters; +performance uses its own smaller context and shared discovery workspace. +Before selecting the measured coverage revision, the job stages only the tested workflow's +two helper script blobs and a revision marker outside the discovery tree in `RUNNER_TEMP`. +It then checks out the exact base or tested head and provisions fixtures there. An old base need +not contain the new helper: `source-revision.txt` identifies the measured Rust checkout, while +`helper_revision` records the separately staged control code and must match the tested workflow. +Missing helpers, marker, or provenance fail closed; this does not copy head Rust source or binaries +into the base measurement, or introduce another discoverable environment. + +Coverage measurement artifacts include `source-revision.txt` and `measurement-context.json`, +LCOV, production reports, and subprocess proof. Artifact names distinguish revision and workflow +attempt; partial reruns cannot silently reuse the other revision's earlier measurement. The gate +requires successful measurements/proofs, the expected source SHAs, and matching nonempty runner +image, architecture, toolchain/LLVM, tool, feature, and fixture-interpreter versions before the +unchanged numerical comparison. Missing or incompatible evidence is an invalid snapshot, not an +improvement or a reason to fall back to an older successful artifact. Separate coverage machines +are not claimed to have identical hardware; observable context mismatches remain blocking. + +Performance builds each revision and its benchmark harness in its own source tree before timing, +then measures the exact base and PR sequentially on the same runner. The detached base worktree +is outside the discovery workspace; both harnesses inherit the same `GITHUB_WORKSPACE`, PATH, +Conda installation, and workspace `.venv`. Builds explicitly use each source tree's normal +`target` directory because the harness resolves its PET executable there rather than honor +`CARGO_TARGET_DIR`. Each invocation still uses independent PID-specific caches and retains every +cold/warm sample and inventory check. Both benchmark executions and JSON extractions must succeed; +the report retains both actual snapshots, source/context evidence, and benchmark logs. +Manual performance runs resolve the default branch's exact SHA once and measure it the same way. + +Linux fixture jobs explicitly provision Miniconda without creating or activating an additional +environment. macOS x86_64 installed-manager/performance jobs use `macos-15-intel`; ARM64 +jobs use `macos-15`, with matching native Conda installers. Both PR and baseline performance +matrices retain Intel and include a separate native ARM64 lane. Each architecture measures its +own exact base, uses distinct artifacts/comments, and retains the existing macOS budgets. +Host-image inventory changes are +measured on both revisions, never normalized away or accepted by changing inventory schemas. + +The push/manual coverage and performance baseline workflows remain publishers for inspection +and history, but PR gates no longer depend on their historical artifacts. Fresh comparisons add +a real base coverage job and base performance build/measurement per platform. All regression +budgets, sample requirements, inventory checks, and required coverage evidence remain unchanged. ## Performance gate The performance workflow runs 10 paired cache-cold/cache-warm JSON-RPC iterations on Linux, -Windows, and macOS, plus 10 untimed cache-cold diagnostic iterations. A comparison is valid only +Windows, Intel macOS, and ARM64 macOS, plus 10 untimed cache-cold diagnostic iterations. A comparison is valid only when every required distribution has at least five samples, inventories match within the same inventory schema, and both benchmark execution and JSON extraction succeed. diff --git a/scripts/quality_measurement.py b/scripts/quality_measurement.py new file mode 100644 index 00000000..d8f87ece --- /dev/null +++ b/scripts/quality_measurement.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +"""Stage, capture, and validate comparable exact-base quality measurements.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Sequence + +if not Path(__file__).with_name("quality_snapshot.py").is_file(): + raise SystemExit("Missing measurement helper dependency: quality_snapshot.py") + +from quality_snapshot import ( # noqa: E402 + SnapshotError, + load_json, + run_coverage, + run_performance, + write_report, +) + + +RUNNER_FIELDS = ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion") +COMMON_TOOLS = { + "rustc": ("rustc", "-vV"), + "cargo": ("cargo", "--version"), + "python": ("python", "--version"), + "conda": ("conda", "--version"), +} +COVERAGE_TOOLS = { + "cargo-llvm-cov": ("cargo", "llvm-cov", "--version"), + "pipenv": ("pipenv", "--version"), + "pyenv": ("pyenv", "--version"), + "pixi": ("pixi", "--version"), +} +COMMON_INTERPRETERS = ("conda-base", "conda-named", "venv") +COVERAGE_INTERPRETERS = ( + "conda-prefix", "venv2", "pipenv", "pixi-default", "pixi-dev", "pyenv", +) +HELPER_FILES = ("quality_measurement.py", "quality_snapshot.py") +HELPER_REVISION = "helper-revision.txt" + + +def require_revision(value: str, description: str) -> str: + if re.fullmatch(r"[0-9a-f]{40}", value) is None: + raise SnapshotError(f"Missing or invalid {description}") + return value + + +def command_output(*arguments: str, cwd: Path) -> str: + executable = shutil.which(arguments[0]) + if executable is None: + raise SnapshotError(f"Required measurement tool not found: {arguments[0]}") + try: + value = subprocess.check_output( + [executable, *arguments[1:]], cwd=cwd, text=True, encoding="utf-8", + stderr=subprocess.PIPE, + ).strip() + except subprocess.CalledProcessError as error: + raise SnapshotError( + f"Measurement command failed ({arguments[0]}): {error.stderr.strip()}" + ) from error + if not value: + raise SnapshotError(f"Empty measurement output: {arguments[0]}") + return value + + +def source_revision(directory: Path, expected: str) -> str: + require_revision(expected, "expected source revision") + actual = command_output("git", "--no-pager", "rev-parse", "HEAD", cwd=directory) + if actual != expected: + raise SnapshotError(f"Unexpected source revision in {directory}: {actual}") + return actual + + +def helper_revision(expected: str) -> str: + require_revision(expected, "workflow revision") + actual = Path(__file__).with_name(HELPER_REVISION).read_text(encoding="utf-8").strip() + if actual != expected: + raise SnapshotError("Measurement helper is not from the tested workflow revision") + return actual + + +def helper_blob(directory: Path, revision: str, filename: str) -> bytes: + try: + content = subprocess.check_output( + ["git", "--no-pager", "-C", str(directory), "show", f"{revision}:scripts/{filename}"], + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as error: + details = error.stderr.decode("utf-8", errors="replace").strip() + raise SnapshotError(f"Missing helper {filename} at {revision}: {details}") from error + if not content: + raise SnapshotError(f"Empty helper {filename} at {revision}") + return content + + +def run_stage(args: argparse.Namespace) -> int: + source = args.source_dir.resolve() + destination = args.output_dir.resolve() + revision = source_revision(source, args.revision) + if destination.is_relative_to(source): + raise SnapshotError("Measurement helpers must be staged outside the discovery workspace") + files = {name: helper_blob(source, revision, name) for name in HELPER_FILES} + destination.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (destination / name).write_bytes(content) + (destination / HELPER_REVISION).write_text(revision + "\n", encoding="utf-8") + print(f"Staged measurement helpers from {revision}") + return 0 + + +def venv_python(prefix: Path) -> Path: + return prefix / "Scripts" / "python.exe" if os.name == "nt" else prefix / "bin" / "python" + + +def capture_context(args: argparse.Namespace) -> tuple[str, dict[str, Any]]: + if not args.target.strip(): + raise SnapshotError("Measurement target must not be empty") + if args.workload == "coverage" and args.pipenv_prefix is None: + raise SnapshotError("Coverage capture requires the created Pipenv prefix") + source = args.source_dir.resolve() + workspace = args.workspace.resolve() + revision = source_revision(source, args.expected_source) + helper = helper_revision(args.workflow_revision) + runner = {key: os.environ.get(key, "") for key in RUNNER_FIELDS} + if not all(value.strip() for value in runner.values()): + raise SnapshotError("Runner image context must not be empty") + tools = dict(COMMON_TOOLS) + if args.workload == "coverage": + tools.update(COVERAGE_TOOLS) + versions = { + name: command_output(*command, cwd=source if name in ("rustc", "cargo", "cargo-llvm-cov") else workspace) + for name, command in tools.items() + } + interpreters = { + "venv": command_output(str(venv_python(workspace / ".venv")), "--version", cwd=workspace), + "conda-base": command_output("conda", "run", "--name", "base", "python", "--version", cwd=workspace), + "conda-named": command_output( + "conda", "run", "--name", + "test-env1" if args.workload == "coverage" else "perf-test-env", + "python", "--version", cwd=workspace, + ), + } + context: dict[str, Any] = { + "runner": runner, + "installed_target": args.target, + "features": "ci" if args.workload == "coverage" else "ci-perf", + "helper_revision": helper, + "versions": versions, + "interpreters": interpreters, + } + if args.workload == "coverage": + for name, prefix in ( + ("venv2", workspace / ".venv2"), + ("pipenv", args.pipenv_prefix.resolve()), + ): + interpreters[name] = command_output(str(venv_python(prefix)), "--version", cwd=workspace) + for name in ("default", "dev"): + prefix = workspace / ".pixi" / "envs" / name + executable = prefix / "python.exe" if os.name == "nt" else prefix / "bin" / "python" + interpreters[f"pixi-{name}"] = command_output(str(executable), "--version", cwd=workspace) + interpreters["conda-prefix"] = command_output( + "conda", "run", "--prefix", str(workspace / "prefix-envs" / ".conda1"), + "python", "--version", cwd=workspace, + ) + interpreters["pyenv"] = "\n".join(sorted( + command_output("pyenv", "versions", "--bare", cwd=workspace).splitlines() + )) + context["coverage_target"] = "host" + else: + context["discovery_workspace"] = str(workspace) + return revision, context + + +def run_capture(args: argparse.Namespace) -> int: + revision, context = capture_context(args) + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "source-revision.txt").write_text(revision + "\n", encoding="utf-8") + (args.output_dir / "measurement-context.json").write_text( + json.dumps(context, indent=2, sort_keys=True) + "\n", encoding="utf-8", + ) + return 0 + + +def read_context(directory: Path, args: argparse.Namespace) -> dict[str, Any]: + path = directory / "measurement-context.json" + value = load_json(path) + required = { + "runner": RUNNER_FIELDS, + "versions": (*COMMON_TOOLS, *COVERAGE_TOOLS) if args.workload == "coverage" else tuple(COMMON_TOOLS), + "interpreters": ( + (*COMMON_INTERPRETERS, *COVERAGE_INTERPRETERS) + if args.workload == "coverage" else COMMON_INTERPRETERS + ), + } + for group, keys in required.items(): + values = value.get(group) + if not isinstance(values, dict) or any( + not isinstance(values.get(key), str) or not values[key].strip() for key in keys + ): + raise SnapshotError(f"Missing {group} measurement context in {path}") + if ( + value.get("features") != ("ci" if args.workload == "coverage" else "ci-perf") + or value.get("installed_target") != args.target + or value.get("helper_revision") != args.expected_head + ): + raise SnapshotError(f"Incompatible {args.workload} workload or helper revision in {path}") + if args.workload == "coverage": + if value.get("coverage_target") != "host": + raise SnapshotError(f"Incompatible coverage target in {path}") + elif args.workspace is None or value.get("discovery_workspace") != str(args.workspace.resolve()): + raise SnapshotError(f"Incompatible performance discovery workspace in {path}") + return value + + +def run_compare(args: argparse.Namespace) -> int: + title = "Test Coverage Report" if args.workload == "coverage" else "Performance Report" + try: + if not args.target.strip(): + raise SnapshotError("Measurement target must not be empty") + helper_revision(args.expected_head) + if args.workload == "coverage": + if args.measurements_result != "success": + raise SnapshotError("One or more coverage measurements or required proofs failed") + else: + for name in ( + "head_benchmark_outcome", "base_benchmark_outcome", + "head_metrics_outcome", "base_metrics_outcome", + ): + if getattr(args, name) != "success": + raise SnapshotError(f"Required benchmark or JSON extraction failed: {name}") + for directory, expected in ( + (args.current_dir, args.expected_head), (args.baseline_dir, args.expected_base), + ): + require_revision(expected, "expected source revision") + path = directory / "source-revision.txt" + if path.read_text(encoding="utf-8").strip() != expected: + raise SnapshotError(f"Unexpected source revision in {path}") + current = read_context(args.current_dir, args) + baseline = read_context(args.baseline_dir, args) + if current != baseline: + raise SnapshotError( + f"{args.workload.capitalize()} runner, tool, or fixture versions differ; inspect both measurement contexts" + ) + except (OSError, UnicodeError, SnapshotError) as error: + print(f"Invalid {args.workload} measurement: {error}", file=sys.stderr) + write_report( + f"## {title} ({args.platform})\n\n:x: **Invalid snapshot:** {error}\n", + args.report, args.summary, + ) + return 1 + filename = "lcov.info" if args.workload == "coverage" else "metrics.json" + comparator = run_coverage if args.workload == "coverage" else run_performance + return comparator(argparse.Namespace( + current=args.current_dir / filename, baseline=args.baseline_dir / filename, + platform=args.platform, report=args.report, summary=args.summary, + )) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + stage = commands.add_parser("stage", help="Export the tested workflow's helpers before checking out old source") + stage.add_argument("--source-dir", type=Path, required=True) + stage.add_argument("--output-dir", type=Path, required=True) + stage.add_argument("--revision", required=True) + stage.set_defaults(handler=run_stage) + capture = commands.add_parser("capture", help="Record a revision's tool and fixture context") + capture.add_argument("workload", choices=("coverage", "performance")) + capture.add_argument("--source-dir", type=Path, required=True) + capture.add_argument("--workspace", type=Path, required=True) + capture.add_argument("--output-dir", type=Path, required=True) + capture.add_argument("--target", required=True) + capture.add_argument("--expected-source", required=True) + capture.add_argument("--workflow-revision", required=True) + capture.add_argument("--pipenv-prefix", type=Path) + capture.set_defaults(handler=run_capture) + compare = commands.add_parser("compare", help="Validate provenance and run the unchanged numerical gate") + compare.add_argument("workload", choices=("coverage", "performance")) + compare.add_argument("--current-dir", type=Path, required=True) + compare.add_argument("--baseline-dir", type=Path, required=True) + compare.add_argument("--expected-head", required=True) + compare.add_argument("--expected-base", required=True) + compare.add_argument("--target", required=True) + compare.add_argument("--workspace", type=Path) + compare.add_argument("--platform", required=True) + compare.add_argument("--report", type=Path, required=True) + compare.add_argument("--summary", type=Path) + compare.add_argument("--measurements-result") + for name in ("head-benchmark-outcome", "base-benchmark-outcome", "head-metrics-outcome", "base-metrics-outcome"): + compare.add_argument("--" + name) + compare.set_defaults(handler=run_compare) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return args.handler(args) + except (OSError, UnicodeError, SnapshotError) as error: + print(f"Invalid measurement helper operation: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_quality_workflows.py b/scripts/tests/test_quality_workflows.py index c467f5eb..84f8a6dd 100644 --- a/scripts/tests/test_quality_workflows.py +++ b/scripts/tests/test_quality_workflows.py @@ -1,12 +1,27 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import copy +import json +import os import re +import shutil +import subprocess +import sys +import tempfile import unittest from pathlib import Path +from unittest.mock import patch + +from test_quality_snapshot import performance_snapshot, write_lcov ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) + +import quality_measurement as measurement # noqa: E402 +from quality_snapshot import SnapshotError # noqa: E402 + SAME_REPOSITORY_PR = ( "github.event.pull_request.head.repo.full_name == github.repository" ) @@ -38,7 +53,150 @@ def step_property(step: str, name: str) -> str | None: return None +def copy_measurement_helpers(directory: Path, revision: str) -> Path: + directory.mkdir() + for filename in ("quality_measurement.py", "quality_snapshot.py"): + shutil.copyfile(ROOT / "scripts" / filename, directory / filename) + (directory / "helper-revision.txt").write_text(revision + "\n", encoding="utf-8") + return directory / "quality_measurement.py" + + class QualityWorkflowTests(unittest.TestCase): + def test_linux_conda_setup_does_not_create_or_activate_an_extra_environment(self) -> None: + for workflow in ( + "pr-check.yml", "coverage.yml", "coverage-baseline.yml", + "perf-tests.yml", "perf-baseline.yml", + ): + with self.subTest(workflow=workflow): + path = ".github/workflows/" + workflow + steps = workflow_steps(path) + setup = steps["Set up Conda (Linux)"] + self.assertRegex(setup, r"conda-incubator/setup-miniconda@[0-9a-f]{40}\b") + for setting in ( + 'activate-environment: ""', "auto-activate: false", "run-init: false", + ): + self.assertIn(setting, setup) + self.assertNotIn("$CONDA/condabin", (ROOT / path).read_text(encoding="utf-8")) + self.assertIn("conda info --all", steps["Check Conda version"]) + + def test_macos_runners_keep_both_native_architecture_workloads(self) -> None: + expected = { + "pr-check.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ] * 2, + "build.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ], + "perf-tests.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ], + "perf-baseline.yml": [ + ("macos-15-intel", "x86_64-apple-darwin"), + ("macos-15", "aarch64-apple-darwin"), + ], + } + for workflow, targets in expected.items(): + with self.subTest(workflow=workflow): + text = (ROOT / ".github" / "workflows" / workflow).read_text(encoding="utf-8") + self.assertEqual( + re.findall(r"- os: (macos-\S+)\n\s+target: (\S+)", text), + targets, + ) + self.assertNotIn("macos-14", text) + if workflow != "build.yml": + self.assertIn("CONDA_ARCH: ${{ matrix.conda_arch }}", text) + self.assertIn('if [[ "$(uname -m)" != "$CONDA_ARCH" ]]', text) + self.assertIn("Miniconda3-latest-MacOSX-${CONDA_ARCH}.sh", text) + coverage = (ROOT / ".github/workflows/coverage-macos.yml").read_text(encoding="utf-8") + self.assertIn("runs-on: macos-15\n", coverage) + self.assertNotIn("macos-14", coverage) + + def test_coverage_pairs_clean_exact_revision_jobs_from_the_same_attempt(self) -> None: + path = ".github/workflows/coverage.yml" + text = (ROOT / path).read_text(encoding="utf-8") + steps = workflow_steps(path) + self.assertIn("revision: [base, head]", text) + self.assertIn( + "ref: ${{ matrix.revision == 'base' && github.event.pull_request.base.sha || github.sha }}", + text, + ) + self.assertIn("needs: measurements", text) + self.assertIn("MEASUREMENTS_RESULT: ${{ needs.measurements.result }}", text) + self.assertIn("ref: ${{ github.sha }}", steps["Checkout Workflow"]) + self.assertLess( + text.index("- name: Stage Coverage Measurement Helper"), + text.index("- name: Checkout Measurement Source"), + ) + self.assertIn( + '--output-dir "$RUNNER_TEMP/quality-tools"', steps["Stage Coverage Measurement Helper"], + ) + self.assertIn( + '--revision "$GITHUB_SHA"', steps["Stage Coverage Measurement Helper"], + ) + self.assertIn( + '"$RUNNER_TEMP/quality-tools/quality_measurement.py" capture coverage', + steps["Capture Coverage Measurement Context"], + ) + self.assertIn('--expected-source "$MEASURED_SHA"', steps["Capture Coverage Measurement Context"]) + self.assertIn('--workflow-revision "$GITHUB_SHA"', steps["Capture Coverage Measurement Context"]) + self.assertNotIn("python - <<", text) + self.assertIn('prefix="$(pipenv --venv)"', steps["Create a Pipenv Environment"]) + self.assertLess( + text.index("- name: Create a Pipenv Environment"), + text.index("- name: Create .venv"), + ) + self.assertIn( + "CREATED_PIPENV_PREFIX: ${{ steps.pipenv.outputs.prefix }}", + steps["Capture Coverage Measurement Context"], + ) + upload = steps["Upload Coverage Measurement"] + self.assertIn("${{ matrix.revision }}-${{ github.run_attempt }}", upload) + for step, revision in ( + ("Download Head Coverage Measurement", "head"), + ("Download Exact PR Base Coverage", "base"), + ): + self.assertIn(f"-{revision}-${{{{ github.run_attempt }}}}", steps[step]) + self.assertNotIn("wait_for_baseline.py", text) + self.assertNotIn("dawidd6/action-download-artifact", text) + + def test_performance_builds_each_harness_before_timing_a_shared_workspace(self) -> None: + path = ".github/workflows/perf-tests.yml" + text = (ROOT / path).read_text(encoding="utf-8") + steps = workflow_steps(path) + prepare = steps["Prepare Exact Base Worktree"] + self.assertIn('git worktree add --detach "$RUNNER_TEMP/perf-base" "$base"', prepare) + self.assertIn('base="$PR_BASE"', prepare) + self.assertIn("refs/remotes/origin/$DEFAULT_BRANCH", prepare) + build = steps["Build Release and Benchmark Harnesses"] + self.assertIn('for source in "$RUNNER_TEMP/perf-base" "$GITHUB_WORKSPACE"', build) + self.assertIn("--no-run", build) + for step in (build, steps["Run Exact Base Performance Tests"], steps["Run Performance Tests"]): + self.assertIn("--target-dir target", step) + self.assertIn("--features ci-perf", step) + self.assertLess( + text.index("- name: Build Release and Benchmark Harnesses"), + text.index("- name: Run Exact Base Performance Tests"), + ) + self.assertIn('cd "$RUNNER_TEMP/perf-base"', steps["Run Exact Base Performance Tests"]) + self.assertNotIn("CARGO_TARGET_DIR:", text) + self.assertNotIn("GITHUB_WORKSPACE:", text) + self.assertNotIn("wait_for_baseline.py", text) + self.assertNotIn("dawidd6/action-download-artifact", text) + self.assertIn("comment_header: perf-macos-arm64", text) + self.assertIn("comment_header: perf-macos\n", text) + self.assertIn("capture performance", steps["Capture Performance Measurement Context"]) + self.assertIn('--source-dir "$RUNNER_TEMP/perf-base"', steps["Capture Performance Measurement Context"]) + self.assertIn('--expected-source "$BASE_SHA"', steps["Capture Performance Measurement Context"]) + self.assertNotIn("python - <<", text) + for step in ("Extract Performance Metrics", "Extract Exact Base Performance Metrics"): + self.assertIn("always()", step_property(steps[step], "if")) + self.assertIn("python -m json.tool", steps[step]) + for artifact in ("baseline-perf/", "measurement-context.json", "source-revision.txt", "perf-output.txt"): + self.assertIn(artifact, steps["Upload PR Performance Results"]) + def test_comment_steps_are_fork_safe_and_non_gating(self) -> None: comment_steps = [] for path in (ROOT / ".github" / "workflows").glob("*.yml"): @@ -130,5 +288,287 @@ def test_report_artifacts_are_uploaded_after_comparison(self) -> None: self.assertIsNone(step_property(steps[comparison], "continue-on-error")) +class MeasurementComparisonTests(unittest.TestCase): + def run_measurement(self, kind, *, regression=False, invalid=None, outcomes=None): + baseline_name = "baseline-coverage" if kind == "coverage" else "baseline-perf" + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) + root = parent / "workspace" + root.mkdir() + helper = copy_measurement_helpers(parent / "tools", "a" * 40) + baseline = root / baseline_name + baseline.mkdir() + context = { + "runner": dict.fromkeys( + ("RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"), "runner-version" + ), + "installed_target": "x86_64-unknown-linux-musl", + "features": "ci" if kind == "coverage" else "ci-perf", + "helper_revision": "a" * 40, + "versions": dict.fromkeys(("rustc", "cargo", "python", "conda"), "tool-version"), + "interpreters": dict.fromkeys( + ("conda-base", "conda-named", "venv"), "Python 3.12.1" + ), + } + if kind == "coverage": + context["coverage_target"] = "host" + context["versions"].update(dict.fromkeys( + ("cargo-llvm-cov", "pipenv", "pyenv", "pixi"), "tool-version" + )) + context["interpreters"].update(dict.fromkeys( + ("conda-prefix", "venv2", "pipenv", "pixi-default", "pixi-dev", "pyenv"), + "Python 3.12.1", + )) + for path, hits in ((root, 8000 if regression else 9000), (baseline, 9000)): + write_lcov( + path / "lcov.info", lines_hit=hits, lines_found=10000, + functions_hit=900, functions_found=1000, + ) + else: + context["discovery_workspace"] = str(root.resolve()) + for path, startup in ((root, 5000 if regression else 20), (baseline, 20)): + snapshot = performance_snapshot( + schema_version=3, inventory_schema_version=2, startup_p95=startup, + ) + (path / "metrics.json").write_text(json.dumps(snapshot), encoding="utf-8") + current_context = copy.deepcopy(context) + base_context = copy.deepcopy(context) + if invalid == "tool": + base_context["versions"]["rustc"] = "different compiler" + elif invalid == "image": + base_context["runner"]["ImageVersion"] = "different runner image" + elif invalid == "fixture": + base_context["interpreters"]["venv"] = "Python 3.13.1" + elif invalid == "incomplete": + del current_context["versions"]["rustc"] + del base_context["versions"]["rustc"] + elif invalid == "workload": + current_context["features"] = base_context["features"] = "other-features" + elif invalid == "helper": + base_context["helper_revision"] = "b" * 40 + elif invalid == "coverage-only": + del current_context["versions"]["pixi"] + del base_context["versions"]["pixi"] + elif invalid not in ( + None, "missing", "malformed", "revision", "missing-helper", "missing-dependency", + ): + raise ValueError(f"Unknown invalid measurement case: {invalid}") + (root / "measurement-context.json").write_text(json.dumps(current_context), encoding="utf-8") + (baseline / "measurement-context.json").write_text(json.dumps(base_context), encoding="utf-8") + (root / "source-revision.txt").write_text("a" * 40 + "\n", encoding="utf-8") + (baseline / "source-revision.txt").write_text( + ("a" if invalid == "revision" else "b") * 40 + "\n", encoding="utf-8" + ) + if invalid == "missing": + (baseline / "measurement-context.json").unlink() + elif invalid == "malformed": + (baseline / "measurement-context.json").write_text("{", encoding="utf-8") + elif invalid == "missing-helper": + (helper.parent / "helper-revision.txt").unlink() + elif invalid == "missing-dependency": + (helper.parent / "quality_snapshot.py").unlink() + environment = { + **os.environ, + "PYTHONPATH": str(ROOT / "scripts"), + "PYTHONDONTWRITEBYTECODE": "1", + } + results = { + "MEASUREMENTS_RESULT": "success", + **dict.fromkeys(( + "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", + "HEAD_METRICS_OUTCOME", "BASE_METRICS_OUTCOME", + ), "success"), + **(outcomes or {}), + } + report_path = root / ( + "coverage-report.md" if kind == "coverage" else "performance-report.md" + ) + arguments = [ + sys.executable, "-B", str(helper), "compare", kind, + "--current-dir", str(root), "--baseline-dir", str(baseline), + "--expected-head", "a" * 40, "--expected-base", "b" * 40, + "--target", "x86_64-unknown-linux-musl", "--workspace", str(root), + "--platform", "Linux", "--report", str(report_path), + "--summary", str(root / "summary.md"), + ] + for variable, flag in ( + ("MEASUREMENTS_RESULT", "--measurements-result"), + ("HEAD_BENCHMARK_OUTCOME", "--head-benchmark-outcome"), + ("BASE_BENCHMARK_OUTCOME", "--base-benchmark-outcome"), + ("HEAD_METRICS_OUTCOME", "--head-metrics-outcome"), + ("BASE_METRICS_OUTCOME", "--base-metrics-outcome"), + ): + arguments.extend((flag, results[variable])) + result = subprocess.run( + arguments, cwd=root, env=environment, + capture_output=True, text=True, check=False, + ) + if invalid == "missing-dependency": + self.assertFalse(report_path.exists()) + return result, "" + self.assertTrue(report_path.is_file(), result.stderr) + return result, report_path.read_text(encoding="utf-8") + + def test_complete_measurements_retain_the_numerical_regression_gates(self) -> None: + for kind in ("coverage", "performance"): + for regression in (False, True): + with self.subTest(kind=kind, regression=regression): + result, report = self.run_measurement(kind, regression=regression) + self.assertEqual(result.returncode, int(regression), result.stderr) + self.assertNotIn("Invalid snapshot", report) + self.assertIn( + "Regression detected" if regression else "Within regression budget", report + ) + + def test_invalid_provenance_or_context_cannot_pass_unchanged_snapshots(self) -> None: + for kind in ("coverage", "performance"): + for invalid in ( + "missing", "malformed", "incomplete", "tool", + "image", "fixture", "workload", "revision", "helper", "missing-helper", + ): + with self.subTest(kind=kind, invalid=invalid): + result, report = self.run_measurement(kind, invalid=invalid) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Invalid snapshot", report) + self.assertNotIn("Within regression budget", report) + + def test_failed_execution_or_extraction_cannot_pass_complete_snapshots(self) -> None: + for variable in ( + "MEASUREMENTS_RESULT", "HEAD_BENCHMARK_OUTCOME", "BASE_BENCHMARK_OUTCOME", + "HEAD_METRICS_OUTCOME", "BASE_METRICS_OUTCOME", + ): + kind = "coverage" if variable == "MEASUREMENTS_RESULT" else "performance" + with self.subTest(variable=variable): + result, report = self.run_measurement(kind, outcomes={variable: "failure"}) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Invalid snapshot", report) + self.assertNotIn("Within regression budget", report) + + def test_missing_staged_dependency_cannot_fall_back_to_pythonpath(self) -> None: + result, _ = self.run_measurement("coverage", invalid="missing-dependency") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Missing measurement helper dependency", result.stderr) + + def test_coverage_still_requires_its_manager_specific_context(self) -> None: + result, report = self.run_measurement("coverage", invalid="coverage-only") + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("Missing versions measurement context", report) + + +class MeasurementCaptureTests(unittest.TestCase): + def test_capture_keeps_base_source_separate_from_helper_and_discovery_workspace(self) -> None: + for workload in ("coverage", "performance"): + with self.subTest(workload=workload), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "old-base" + workspace = root / "workspace" + source.mkdir() + workspace.mkdir() + helper = copy_measurement_helpers(root / "tools", "a" * 40) + output = workspace / "baseline" + arguments = [ + "capture", workload, "--source-dir", str(source), + "--workspace", str(workspace), "--output-dir", str(output), + "--target", "aarch64-apple-darwin", "--expected-source", "b" * 40, + "--workflow-revision", "a" * 40, + ] + if workload == "coverage": + arguments.extend(("--pipenv-prefix", str(workspace / "created-pipenv"))) + calls = [] + + def command(*args, cwd): + calls.append((args, cwd)) + return "b" * 40 if args[0] == "git" else "version" + + with ( + patch.object(measurement, "__file__", str(helper)), + patch.object(measurement, "command_output", side_effect=command), + patch.dict(os.environ, dict.fromkeys(measurement.RUNNER_FIELDS, "runner")), + ): + self.assertEqual(measurement.main(arguments), 0) + context = json.loads((output / "measurement-context.json").read_text(encoding="utf-8")) + self.assertEqual((output / "source-revision.txt").read_text().strip(), "b" * 40) + self.assertEqual(context["helper_revision"], "a" * 40) + self.assertFalse((source / "scripts" / "quality_measurement.py").exists()) + for args, cwd in calls: + self.assertEqual( + cwd, source.resolve() if args[0] in ("git", "rustc", "cargo") else workspace.resolve() + ) + if workload == "coverage": + self.assertIn("pipenv", context["versions"]) + self.assertIn("pixi-dev", context["interpreters"]) + self.assertIn("coverage_target", context) + self.assertTrue(any("test-env1" in args for args, _ in calls)) + else: + self.assertNotIn("pipenv", context["versions"]) + self.assertNotIn("pixi-dev", context["interpreters"]) + self.assertEqual(context["discovery_workspace"], str(workspace.resolve())) + self.assertTrue(any("perf-test-env" in args for args, _ in calls)) + + def test_capture_rejects_head_reused_as_base_before_writing_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / "evidence" + arguments = [ + "capture", "performance", "--source-dir", str(root), + "--workspace", str(root), "--output-dir", str(output), + "--target", "aarch64-apple-darwin", "--expected-source", "b" * 40, + "--workflow-revision", "a" * 40, + ] + with patch.object(measurement, "command_output", return_value="a" * 40): + self.assertEqual(measurement.main(arguments), 1) + self.assertFalse(output.exists()) + + def test_stage_exports_immutable_helpers_without_copying_measured_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "workflow" + source.mkdir() + destination = root / "tools" + files = { + name: (ROOT / "scripts" / name).read_bytes() + for name in ("quality_measurement.py", "quality_snapshot.py") + } + arguments = [ + "stage", "--source-dir", str(source), "--output-dir", str(destination), + "--revision", "a" * 40, + ] + with ( + patch.object(measurement, "command_output", return_value="a" * 40), + patch.object( + measurement, "helper_blob", + side_effect=lambda path, revision, filename: files[filename], + ) as blobs, + ): + self.assertEqual(measurement.main(arguments), 0) + for name, content in files.items(): + self.assertEqual((destination / name).read_bytes(), content) + self.assertEqual((destination / "helper-revision.txt").read_text().strip(), "a" * 40) + self.assertEqual(list(source.iterdir()), []) + self.assertEqual(blobs.call_count, 2) + self.assertTrue(all(call.args[1] == "a" * 40 for call in blobs.call_args_list)) + + def test_stage_rejects_missing_helpers_and_workspace_injection(self) -> None: + for invalid in ("revision", "missing-helper", "inside-workspace"): + with self.subTest(invalid=invalid), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "workflow" + source.mkdir() + destination = source / "tools" if invalid == "inside-workspace" else root / "tools" + arguments = [ + "stage", "--source-dir", str(source), "--output-dir", str(destination), + "--revision", "a" * 40, + ] + with ( + patch.object( + measurement, "command_output", + return_value=("b" if invalid == "revision" else "a") * 40, + ), + patch.object(measurement, "helper_blob", side_effect=SnapshotError("Missing helper")), + ): + self.assertEqual(measurement.main(arguments), 1) + self.assertFalse((destination / "helper-revision.txt").exists()) + + if __name__ == "__main__": unittest.main()