Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions .github/actions/prepare-upgrade-source/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: Prepare upgrade source
description: >-
Download a published release's package and runtime images under
artifacts/upgrade, the baseline of the upgrade disruption. Run it in the job
that uses them, so a failure to download affects only that job.
inputs:
tag:
description: Release tag to use as the baseline, such as v0.1.9
required: true
package:
description: Package type of the baseline; only deb is supported
required: false
default: deb
github-token:
description: Token that reads the release and pulls its images from GHCR
required: true

runs:
using: composite
steps:
# The package doesn't embed the images; its gateway pins the release tag.
# They go in as tarballs so the guest needs no registry access.
- name: Download the released package
shell: bash
env:
GH_TOKEN: ${{ inputs.github-token }}
SOURCE_TAG: ${{ inputs.tag }}
PACKAGE: ${{ inputs.package }}
run: |
set -euo pipefail
if [ -z "${SOURCE_TAG}" ]; then
echo "::error::no release tag to use as the upgrade baseline"
exit 1
fi
if [ "${PACKAGE}" != "deb" ]; then
echo "::error::unsupported upgrade source package type: ${PACKAGE}"
exit 1
fi
source_dir=artifacts/upgrade/deb/source
download_dir="${source_dir}/download"
gh release download "$SOURCE_TAG" --repo "$GITHUB_REPOSITORY" \
--pattern 'openshell_*_amd64.deb' \
--pattern openshell-checksums-sha256.txt \
--dir "$download_dir"
deb_path=$(find "$download_dir" -maxdepth 1 -type f -name 'openshell_*_amd64.deb' -print -quit)
if [[ -z "$deb_path" || ! -f "${download_dir}/openshell-checksums-sha256.txt" ]]; then
echo "::error::release ${SOURCE_TAG} did not contain the expected Debian package and checksums"
exit 1
fi
(
cd "$download_dir"
grep -F " $(basename "$deb_path")" openshell-checksums-sha256.txt | sha256sum --check --status
)
mv "$deb_path" "${source_dir}/openshell.deb"
rm -rf "$download_dir"
- name: Export the released runtime images
shell: bash
env:
GH_TOKEN: ${{ inputs.github-token }}
SOURCE_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
echo "${GH_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
image_tag="${SOURCE_TAG#v}"
mkdir -p artifacts/upgrade/source-images
for image in sandbox supervisor; do
docker pull "ghcr.io/nvidia/openshell/${image}:${image_tag}"
docker save --output "artifacts/upgrade/source-images/openshell-${image}-tmachine.tar" \
"ghcr.io/nvidia/openshell/${image}:${image_tag}"
done
2 changes: 2 additions & 0 deletions .github/workflows/branch-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ jobs:
cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
cargo fmt --manifest-path tests/suites/conformance/Cargo.toml --all -- --check
cargo fmt --manifest-path tests/suites/continuity/Cargo.toml --all -- --check
cargo fmt --manifest-path tests/suites/drivers/podman/Cargo.toml --all -- --check
cargo fmt --manifest-path tests/suites/features/Cargo.toml --all -- --check
cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check
Expand Down Expand Up @@ -359,6 +360,7 @@ jobs:
cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --all-targets -- -D warnings
cargo clippy --locked --manifest-path tests/suites/continuity/Cargo.toml --workspace --all-targets -- -D warnings
cargo clippy --locked --manifest-path tests/suites/drivers/podman/Cargo.toml --workspace --all-targets -- -D warnings
cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml --workspace --all-targets -- -D warnings
cargo clippy --locked --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings
Expand Down
24 changes: 24 additions & 0 deletions .github/workflows/branch-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ jobs:
run_gpu_e2e: ${{ steps.labels.outputs.run_gpu_e2e }}
run_kubernetes_ha_e2e: ${{ steps.labels.outputs.run_kubernetes_ha_e2e }}
run_kubernetes_credential_drivers_e2e: ${{ steps.labels.outputs.run_kubernetes_credential_drivers_e2e }}
run_continuity_e2e: ${{ steps.labels.outputs.run_continuity_e2e }}
run_any_e2e: ${{ steps.labels.outputs.run_any_e2e }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -47,12 +48,14 @@ jobs:
run_gpu_e2e="$(jq -r 'index("test:e2e-gpu") != null' <<< "$LABELS_JSON")"
run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
run_kubernetes_credential_drivers_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
run_continuity_e2e="$(jq -r 'index("test:e2e:continuity") != null' <<< "$LABELS_JSON")"
;;
*)
run_core_e2e=true
run_gpu_e2e=true
run_kubernetes_ha_e2e=false
run_kubernetes_credential_drivers_e2e=false
run_continuity_e2e=true
;;
esac
if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_kubernetes_credential_drivers_e2e" = "true" ]; then
Expand All @@ -66,6 +69,7 @@ jobs:
echo "run_gpu_e2e=$run_gpu_e2e"
echo "run_kubernetes_ha_e2e=$run_kubernetes_ha_e2e"
echo "run_kubernetes_credential_drivers_e2e=$run_kubernetes_credential_drivers_e2e"
echo "run_continuity_e2e=$run_continuity_e2e"
echo "run_any_e2e=$run_any_e2e"
} >> "$GITHUB_OUTPUT"
Expand Down Expand Up @@ -463,6 +467,26 @@ jobs:
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

# Run the continuity suites (tests/suites/continuity) on tmachine: state
# seeded before a disruption, such as a gateway restart or an upgrade from the
# latest release, must survive it. The label adds to test:e2e, whose builds it
# reuses.
continuity-integration:
needs: [pr_metadata, prepare-integration]
if: needs.pr_metadata.outputs.run_integration == 'true' && needs.pr_metadata.outputs.run_continuity_e2e == 'true'
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-suites.yml
with:
family: continuity
upgrade-source: true
source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

# Run the Podman e2e suite on tmachine against the packaged gateway.
podman-e2e-integration:
needs: prepare-integration
Expand Down
7 changes: 6 additions & 1 deletion .github/workflows/e2e-label-help.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ permissions: {}
jobs:
hint:
name: Post next-step hint for E2E label
if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes'
if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes' || github.event.label.name == 'test:e2e:continuity'
runs-on: ubuntu-latest
permissions:
pull-requests: write
Expand Down Expand Up @@ -55,6 +55,11 @@ jobs:
build_summary="gateway, sandbox, and supervisor images"
status_summary="This is an optional proof-of-life suite; failures are visible in the workflow run but do not publish a required CI gate status."
;;
test:e2e:continuity)
suite_summary="the continuity suites (gateway restart and upgrade from the latest release); apply test:e2e as well, since it builds what they run on"
build_summary="gateway, sandbox, and supervisor images and packages"
status_summary="This is an optional suite; failures are visible in the workflow run but do not publish a required CI gate status."
;;
*) echo "Unrecognized label $LABEL_NAME"; exit 1 ;;
esac
Expand Down
44 changes: 41 additions & 3 deletions .github/workflows/e2e-suites.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ on:
required: false
type: boolean
default: false
upgrade-source:
description: Also run the rows that upgrade from a released package; each downloads the latest release as its baseline
required: false
type: boolean
default: false
prebuilt:
description: Run the rows with this run's prebuilt tests instead of building them in every job; conformance rows take the openshell-conformance-x86_64-unknown-linux-musl artifact, other rows their family's openshell-e2e-bundle-<family>-x86_64-unknown-linux-musl artifact
required: false
Expand Down Expand Up @@ -71,12 +76,14 @@ jobs:
timeout-minutes: 5
outputs:
matrix: ${{ steps.rows.outputs.matrix }}
upgrade_source_tag: ${{ steps.baseline.outputs.tag }}
steps:
- name: Select the rows to run
id: rows
env:
FAMILY: ${{ inputs.family }}
EXTERNAL_DRIVERS: ${{ inputs.external-drivers }}
UPGRADE_SOURCE: ${{ inputs.upgrade-source }}
run: |
set -euo pipefail
# Only amd64 for now: CI has no arm64 runner with KVM for tmachine,
Expand Down Expand Up @@ -114,14 +121,18 @@ jobs:
{"environment":"fedora-podman-rootless-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless-userns-auto","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless-userns-keep-id","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless-userns-private","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"}
{"environment":"fedora-podman-rootless-userns-private","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"continuity","disruption":"gateway-restart","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","family":"continuity","disruption":"upgrade","gateway":"openshell","runner":"ubuntu-24.04","upgrade_source":true,"upgrade_package":"deb"}
]'
# `suite` names the row's tests in the job and artifact names.
matrix=$(jq -c --arg family "${FAMILY}" --arg external "${EXTERNAL_DRIVERS}" '{include: map(
matrix=$(jq -c --arg family "${FAMILY}" --arg external "${EXTERNAL_DRIVERS}" --arg upgrade "${UPGRADE_SOURCE}" '{include: map(
select($family == "" or .family == $family or (.family | startswith($family + "/")))
| select((.external_driver // false) == false or $external == "true")
| select((.upgrade_source // false) == false or $upgrade == "true")
| .package //= ""
| .suite = (if .package == "" then .family else "\(.family)/\(.package)" end)
| .disruption //= ""
| .suite = ([.family, .package, .disruption] | map(select(. != "")) | join("/"))
| .artifact = ([.suite, .environment, .installer] | join("-") | gsub("/"; "-"))
| .bundle = "openshell-e2e-bundle-\(.family | gsub("/"; "-"))-x86_64-unknown-linux-musl"
)}' <<< "$rows")
Expand All @@ -132,6 +143,18 @@ jobs:
echo "$matrix"
echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
# Resolve the latest release once, so every upgrade row tests against the
# same baseline. A failure leaves the tag empty and fails only the upgrade
# rows, instead of every row.
- name: Resolve the upgrade baseline
id: baseline
if: inputs.upgrade-source
env:
GH_TOKEN: ${{ github.token }}
run: |
tag=$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName || true)
echo "tag=${tag}" >> "$GITHUB_OUTPUT"
suites:
name: E2E suites (${{ matrix.suite }}, ${{ matrix.environment }}, ${{ matrix.installer }})
needs: select
Expand Down Expand Up @@ -193,6 +216,16 @@ jobs:
artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }}
path: artifacts

# The baseline of an upgrade row. It is prepared here, not in the shared
# input preparation, so a failure to fetch it affects only this row.
- name: Prepare the upgrade baseline
if: matrix.upgrade_source
uses: ./.github/actions/prepare-upgrade-source
with:
tag: ${{ needs.select.outputs.upgrade_source_tag }}
package: ${{ matrix.upgrade_package }}
github-token: ${{ github.token }}

# The program and the bundles are built once per run (see
# build-conformance and build-e2e-bundles in branch-e2e.yml), so the rows
# need not each build the suites.
Expand All @@ -218,6 +251,7 @@ jobs:
FAMILY: ${{ matrix.family }}
PACKAGE: ${{ matrix.package }}
GATEWAY: ${{ matrix.gateway }}
DISRUPTION: ${{ matrix.disruption }}
run: |
# The tests this environment can't pass yet; the runner passes the
# variable on to the guest.
Expand All @@ -244,6 +278,10 @@ jobs:
selection+=(--package "${PACKAGE}")
fi
fi
# A continuity row seeds state, applies the disruption, and verifies.
if [ -n "${DISRUPTION}" ]; then
selection+=(--disruption "${DISRUPTION}")
fi
nix run .#openshell-test-runner -- run \
--environment "${ENVIRONMENT}" \
--installer "${INSTALLER}" \
Expand Down
17 changes: 17 additions & 0 deletions .github/workflows/release-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,22 @@ jobs:
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

continuity-integration:
needs: prepare-integration
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-suites.yml
with:
family: continuity
upgrade-source: true
cancel-in-progress: false
source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

feature-specific-integration:
needs: prepare-integration
permissions:
Expand Down Expand Up @@ -262,6 +278,7 @@ jobs:
- compute-versions
- package-binaries
- conformance-integration
- continuity-integration
- feature-specific-integration
- docker-e2e
- vm-e2e
Expand Down
18 changes: 18 additions & 0 deletions .github/workflows/release-qualification.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,21 @@ jobs:
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

continuity-integration:
needs: prepare-integration
permissions:
actions: read
contents: read
packages: read
uses: ./.github/workflows/e2e-suites.yml
with:
family: continuity
upgrade-source: true
source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
secrets:
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}

feature-specific-integration:
needs: prepare-integration
permissions:
Expand Down Expand Up @@ -161,6 +176,7 @@ jobs:
- protobuf-compatibility
- security
- conformance-integration
- continuity-integration
- feature-specific-integration
- docker-e2e
- vm-e2e
Expand All @@ -179,6 +195,7 @@ jobs:
id: summary
env:
CONFORMANCE_RESULT: ${{ needs.conformance-integration.result }}
CONTINUITY_RESULT: ${{ needs.continuity-integration.result }}
DOCKER_E2E_RESULT: ${{ needs.docker-e2e.result }}
FEATURE_INTEGRATION_RESULT: ${{ needs.feature-specific-integration.result }}
IS_PRERELEASE: ${{ inputs.is-prerelease }}
Expand Down Expand Up @@ -210,6 +227,7 @@ jobs:
echo "| Protobuf API compatibility | ${PROTO_COMPATIBILITY_RESULT} |"
echo "| Security | ${SECURITY_RESULT} |"
echo "| Conformance integration | ${CONFORMANCE_RESULT} |"
echo "| Continuity integration | ${CONTINUITY_RESULT} |"
echo "| Feature integration | ${FEATURE_INTEGRATION_RESULT} |"
echo "| Docker E2E | ${DOCKER_E2E_RESULT} |"
echo "| VM E2E | ${VM_E2E_RESULT} |"
Expand Down
Loading
Loading