From f25252475a4b1405f6daee551ce998c6dfb0e0b1 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 19 Nov 2026 10:34:33 -0800 Subject: [PATCH 1/3] ci(core-sync): use the shared core-sync scripts and workflows Replace the core-sync scripts and tag-release.sh with mise remote tasks pinned to ai-openshell/workflows, and turn core-watch.yml and core-drift.yml into callers of the shared reusable workflows. The release workflow tags through the shared release:tag task. This repository keeps the language-specific parts the shared scripts call: proto:buf, core:uncovered-rpcs (gRPC stub parsing) and OPENSHELL_GENERATED_PATHS. The shared drift check also runs check before test, so a type error in bindings from core main now fails it. Draft: pins pimlock/workflows until the shared PR merges. Signed-off-by: Piotr Mlocek --- .github/CODEOWNERS | 4 +- .github/workflows/core-drift.yml | 48 +----- .github/workflows/core-watch.yml | 71 +------- .github/workflows/release.yml | 10 +- CONTRIBUTING.md | 2 +- mise.toml | 32 +++- scripts/README.md | 27 ++- scripts/core-sync/bump-pr-body.md | 16 -- scripts/core-sync/core-tags.sh | 72 -------- scripts/core-sync/drift-check.sh | 131 -------------- scripts/core-sync/drift-issue.sh | 35 ---- scripts/core-sync/open-bump-pr.sh | 220 ------------------------ scripts/core-sync/report-failure.sh | 47 ----- scripts/core-sync/update-drift-issue.sh | 64 ------- scripts/release/tag-release.sh | 127 -------------- scripts/uncovered-rpcs.sh | 18 ++ scripts/vendored-core-files.txt | 2 +- 17 files changed, 77 insertions(+), 849 deletions(-) delete mode 100644 scripts/core-sync/bump-pr-body.md delete mode 100755 scripts/core-sync/core-tags.sh delete mode 100755 scripts/core-sync/drift-check.sh delete mode 100644 scripts/core-sync/drift-issue.sh delete mode 100755 scripts/core-sync/open-bump-pr.sh delete mode 100755 scripts/core-sync/report-failure.sh delete mode 100755 scripts/core-sync/update-drift-issue.sh delete mode 100755 scripts/release/tag-release.sh create mode 100755 scripts/uncovered-rpcs.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4bbf5db..b2b0bf9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,5 @@ # The sdk team owns the SDK; OpenShell maintainers review as a fallback. * @ai-openshell/sdk @ai-openshell/maintainers -# CI and the scripts it runs with the bot token or PyPI access need an infra review. +# CI and what it runs with the bot token or PyPI access need an infra review. /.github/ @ai-openshell/infra -/scripts/core-sync/ @ai-openshell/infra -/scripts/release/ @ai-openshell/infra diff --git a/.github/workflows/core-drift.yml b/.github/workflows/core-drift.yml index 94fc274..3d863b7 100644 --- a/.github/workflows/core-drift.yml +++ b/.github/workflows/core-drift.yml @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # Nightly comparison of core main with the pinned core tag; keeps the single -# "Core main drift" issue current (mise run core:drift). +# "Core main drift" issue current (mise run core:drift). Uses the shared +# workflow from ai-openshell/workflows (core-sync/). Keep the ref in step with +# the core-sync tasks in mise.toml. name: core-drift on: @@ -12,48 +14,6 @@ on: permissions: contents: read -concurrency: - group: core-drift - cancel-in-progress: false - jobs: drift: - name: drift - # The bot secrets live in the sdk-bot environment, which only main can - # deploy to, so a pushed branch can't mint a bot token. - environment: sdk-bot - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - id: app-token - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 - with: - client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} - private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} - permission-issues: write - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 - - # Exit 1 with a report means drift; the issue records it. No report - # means the check itself broke, which the failure step records. - - name: Check drift against core main - run: | - mise run core:drift -- --out drift-report.md || true - test -s drift-report.md - cat drift-report.md >>"$GITHUB_STEP_SUMMARY" - - - name: Update the drift issue - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: mise run core:drift-issue -- drift-report.md - - - name: Report failure - if: failure() - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: mise run core:report-failure -- core-drift "$RUN_URL" + uses: pimlock/workflows/.github/workflows/core-drift.yml@63d4b041b9b99fdfa3491573d31a537451a427ab # feat/core-sync/pimlock diff --git a/.github/workflows/core-watch.yml b/.github/workflows/core-watch.yml index da79469..9f71cc6 100644 --- a/.github/workflows/core-watch.yml +++ b/.github/workflows/core-watch.yml @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 -# Opens one update proposal per new core release tag -# (mise run core:bump). +# Opens one update proposal per new core release tag (mise run core:bump), +# with the shared workflow from ai-openshell/workflows (core-sync/). Keep the +# ref in step with the core-sync tasks in mise.toml. name: core-watch on: @@ -17,68 +18,8 @@ on: permissions: contents: read -concurrency: - group: core-watch - cancel-in-progress: false - jobs: watch: - name: watch - # The bot secrets live in the sdk-bot environment, which only main can - # deploy to, so a pushed branch can't mint a bot token. - environment: sdk-bot - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - id: app-token - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 - with: - client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} - private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} - permission-contents: write - permission-pull-requests: write - permission-issues: write - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - token: ${{ steps.app-token.outputs.token }} - persist-credentials: true - - - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 - - - name: Set the bot identity - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} - run: | - set -euo pipefail - id=$(gh api "users/${APP_SLUG}[bot]" --jq .id) - git config user.name "${APP_SLUG}[bot]" - git config user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com" - - - name: Propose new core tags - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - INPUT_TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - if [ -n "$INPUT_TAG" ]; then - tags=$INPUT_TAG - else - tags=$(mise run --quiet core:tags) - fi - if [ -z "$tags" ]; then - echo "No core tags newer than $(mise run --quiet core:version)." - exit 0 - fi - for t in $tags; do - mise run core:bump -- "$t" - done - - - name: Report failure - if: failure() - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: mise run core:report-failure -- core-watch "$RUN_URL" + uses: pimlock/workflows/.github/workflows/core-watch.yml@63d4b041b9b99fdfa3491573d31a537451a427ab # feat/core-sync/pimlock + with: + tag: ${{ inputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0191316..81cff88 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,6 +41,12 @@ jobs: fetch-depth: 0 persist-credentials: false + # Only for the shared release:tag task (git and gh); no tools needed. + - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 + with: + install: false + cache: false + - id: tag name: Tag the release and create a draft env: @@ -56,11 +62,11 @@ jobs: echo "::error::$sha is not on main" exit 1 } - scripts/release/tag-release.sh --force --sha "$sha" >>"$GITHUB_OUTPUT" + mise run --quiet release:tag -- --force --sha "$sha" >>"$GITHUB_OUTPUT" else # Compare against the commit before the push, so a pin change # that isn't the pushed tip still counts. - scripts/release/tag-release.sh --since "$BEFORE" >>"$GITHUB_OUTPUT" + mise run --quiet release:tag -- --since "$BEFORE" >>"$GITHUB_OUTPUT" fi build: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2cc7e63..fb6285e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,7 @@ Merging a pull request that moves the pin to a stable core tag releases the SDK. changed to a stable tag. Then it: 1. tags `vX.Y.Z` at the merged commit and creates a draft GitHub release, with - the pull request's description as notes (`scripts/release/tag-release.sh`); + the pull request's description as notes (`mise run release:tag`, shared through ai-openshell/workflows); 2. builds the wheel and sdist at version `X.Y.Z`, verifies them, and writes their checksums; 3. publishes to PyPI from the `pypi` environment with trusted publishing; diff --git a/mise.toml b/mise.toml index b30b961..4495552 100644 --- a/mise.toml +++ b/mise.toml @@ -17,8 +17,10 @@ buf = "1.72.0" uv = "0.10.12" [env] -# The single core pin: a release tag. scripts/core-sync/ requires a tag. +# The single core pin: a release tag. The shared core-sync scripts require a tag. OPENSHELL_CORE_REF = "v0.1.9" +# The committed generated bindings, for the core-sync summaries. +OPENSHELL_GENERATED_PATHS = "src/openshell/_proto" # The package version for builds. CI and local builds use 0.0.0; the release # workflow sets OPENSHELL_SDK_VERSION to the release's version. SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL = "{{ get_env(name='OPENSHELL_SDK_VERSION', default='0.0.0') }}" @@ -147,22 +149,40 @@ done description = "Run the full CI pipeline" depends = ["check", "test", "build"] +# --- Core sync -------------------------------------------------------------- +# The scripts are shared through ai-openshell/workflows (core-sync/), pinned to +# the same commit as the reusable workflows in core-watch.yml and +# core-drift.yml; update the refs together. They call the tasks below this +# block (core-sync/README.md there has the contract). + [tasks."core:tags"] description = "List core release tags newer than the pin (newest pre-release only, --all for every tag)" -run = "scripts/core-sync/core-tags.sh" +file = "git::https://github.com/pimlock/workflows.git//core-sync/core-tags.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" [tasks."core:bump"] description = "Open the update proposal for a core tag: mise run core:bump -- [--dry-run]" -run = "scripts/core-sync/open-bump-pr.sh" +file = "git::https://github.com/pimlock/workflows.git//core-sync/open-bump-pr.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" [tasks."core:drift"] description = "Compare core main with the pin: mise run core:drift -- --out " -run = "scripts/core-sync/drift-check.sh" +file = "git::https://github.com/pimlock/workflows.git//core-sync/drift-check.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" [tasks."core:drift-issue"] description = "Create, update, or close the core main drift issue from a report: mise run core:drift-issue -- [--dry-run]" -run = "scripts/core-sync/update-drift-issue.sh" +file = "git::https://github.com/pimlock/workflows.git//core-sync/update-drift-issue.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" [tasks."core:report-failure"] description = "Record a failed automation run in the drift issue: mise run core:report-failure -- [--dry-run]" -run = "scripts/core-sync/report-failure.sh" +file = "git::https://github.com/pimlock/workflows.git//core-sync/report-failure.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" + +[tasks."release:tag"] +description = "Tag and draft the release when a stable pin lands on main: mise run release:tag -- [--dry-run] [--force] [--sha ] [--since ]" +file = "git::https://github.com/pimlock/workflows.git//core-sync/tag-release.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" + +[tasks."proto:buf"] +description = "Run buf: mise run proto:buf -- " +run = "buf" + +[tasks."core:uncovered-rpcs"] +description = "List the RPCs in src/openshell/_proto that the SDK doesn't call, one Service.Method per line" +run = "scripts/uncovered-rpcs.sh" diff --git a/scripts/README.md b/scripts/README.md index e0757b6..870c7d2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,26 +2,23 @@ The workflows under `.github/workflows/` run these scripts, most through a mise task, so each job can be run locally first. Scripts that write to GitHub take `--dry-run` and print what they would do instead. -## `core-sync/`: keeping up with core +## Core sync and release tagging (shared) -Move the core pin (`OPENSHELL_CORE_REF` in `mise.toml`) to new core releases, and watch core `main` for changes that will break the SDK. +The core-sync scripts and the `core-watch` and `core-drift` workflows are shared with the other SDKs through [ai-openshell/workflows](https://github.com/ai-openshell/workflows/tree/main/core-sync), which documents them and the contract this repository implements. `mise.toml` pins the scripts as remote tasks, and `core-watch.yml` and `core-drift.yml` call the shared workflows at the same commit; update the two refs together. -The workflows use a GitHub App token from the `sdk-bot` environment (secrets `SDK_BOT_CLIENT_ID` and `SDK_BOT_PRIVATE_KEY`), which only `main` can deploy to. They never use `GITHUB_TOKEN`, because pushes and PRs made with it don't trigger CI. +| mise task | Runs | +|-----------|------| +| `core:tags`, `core:bump`, `core:drift`, `core:drift-issue`, `core:report-failure`, `release:tag` | The shared scripts. | +| `proto:fetch`, `proto:gen`, `proto:buf`, `check`, `test` | This repository's parts of the contract. | +| `core:uncovered-rpcs` | `uncovered-rpcs.sh`: RPCs in `src/openshell/_proto` the SDK doesn't call, for the drift report. | -| File | mise task | Purpose | -|------|-----------|---------| -| `core-tags.sh [--all]` | `core:tags` | List NVIDIA/OpenShell release tags newer than the pin: every stable tag, plus the newest pre-release when it is newer than all of them. | -| `open-bump-pr.sh [--dry-run]` | `core:bump` | Open the update proposal for one core tag: branch `core/`, pin change, regenerated bindings, refreshed copies of core files, proto diff in the body, labels `core-bump` plus `stable` or `prerelease`. Closes older pre-release proposals. | -| `bump-pr-body.md` | | PR body template for `open-bump-pr.sh`. | -| `drift-check.sh --out ` | `core:drift` | Compare core `main` with the pin: `buf breaking`, regenerate and run the unit tests, list the RPCs the SDK doesn't call. Writes the markdown report. | -| `update-drift-issue.sh [--dry-run]` | `core:drift-issue` | Create, update, or close the single `Core main drift` issue from a report. | -| `report-failure.sh [--dry-run]` | `core:report-failure` | Append a dated failure line to the drift issue. Called from `if: failure()` steps. | -| `drift-issue.sh` | | Shared helpers for the two drift issue scripts (sourced, not run). | - -| Workflow | Runs (mise tasks) | +| Workflow | Runs | |----------|------| | `core-watch.yml` (hourly) | `core:tags`, then `core:bump` per tag | | `core-drift.yml` (nightly) | `core:drift`, then `core:drift-issue` | +| `release.yml` (push to `main`) | `release:tag`; when it prints a tag, builds and publishes the release | + +The shared workflows use a GitHub App token from the `sdk-bot` environment (secrets `SDK_BOT_CLIENT_ID` and `SDK_BOT_PRIVATE_KEY`), which only `main` can deploy to. Bump PRs run E2E against their own pin, because `e2e.yml` defaults to `OPENSHELL_CORE_REF` on the PR branch. @@ -40,5 +37,5 @@ Bump PRs run E2E against their own pin, because `e2e.yml` defaults to `OPENSHELL |------|-----------|---------| | `generate_python_proto.py` | `proto:gen` | Generate `src/openshell/_proto` from `proto/`. | | `verify-python-wheel.py` | `build` | Check the wheel's contents and version. | -| `vendored-core-files.txt` | `vendored:check` | Files copied from core. `vendored:check` compares them with core at the pin; `core-sync/open-bump-pr.sh` refreshes them. | +| `vendored-core-files.txt` | `vendored:check` | Files copied from core. `vendored:check` compares them with core at the pin; `core:bump` refreshes them. | | `fetch-core.sh ...` | `vendored:check` | Copy files from core at a ref. | diff --git a/scripts/core-sync/bump-pr-body.md b/scripts/core-sync/bump-pr-body.md deleted file mode 100644 index cba358c..0000000 --- a/scripts/core-sync/bump-pr-body.md +++ /dev/null @@ -1,16 +0,0 @@ -Move the core pin from `{{OLD}}` to `{{NEW}}` ({{KIND}}). - -- Core changes: [{{OLD}}...{{NEW}}]({{COMPARE_URL}}) -- Regenerated bindings: {{GENERATED}} -- E2E runs against `{{IMAGE}}` -- Files copied from core: -{{FIXTURES}} - -## Proto changes - -{{PROTO_DIFF}} - -## Before merging - -- CI and E2E must pass. If the SDK needs changes for this core version, push them to this branch. -- Pre-release pins are not released; a newer pre-release proposal replaces this one. diff --git a/scripts/core-sync/core-tags.sh b/scripts/core-sync/core-tags.sh deleted file mode 100755 index 8bb4de4..0000000 --- a/scripts/core-sync/core-tags.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -# core-tags.sh: print NVIDIA/OpenShell release tags newer than the pin, -# oldest first. -# -# Usage: core-tags.sh [--all] [--pin ] -# -# Release tags are vX.Y.Z and vX.Y.Z-pre.N; a pre-release sorts below its -# stable version. By default every newer stable tag is printed, plus the -# newest pre-release when it is newer than all of them: one proposal for the -# current release train instead of one per pre-release. --all prints every -# newer tag. --pin overrides the pin, OPENSHELL_CORE_REF in mise.toml. The pin -# must be a release tag; a commit SHA fails. -set -euo pipefail - -repo=${CORE_REPO:-NVIDIA/OpenShell} -all=0 -pin= - -while [ $# -gt 0 ]; do - case $1 in - --all) all=1; shift ;; - --pin) pin=${2:?--pin needs a tag}; shift 2 ;; - *) echo "usage: $0 [--all] [--pin ]" >&2; exit 2 ;; - esac -done - -[ -n "$pin" ] || pin=$(sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p' mise.toml) -[ -n "$pin" ] || { echo "ERROR: no OPENSHELL_CORE_REF in mise.toml" >&2; exit 1; } - -[[ "$pin" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || - { echo "ERROR: pin '$pin' is not a release tag; commit SHA pins are not supported by core-sync" >&2; exit 1; } - -# key : "major minor patch rank pre" with rank 1 for stable, 0 for pre. -key() { - awk -v t="$1" 'BEGIN { - s = t; sub(/^v/, "", s) - pre = ""; rank = 1 - if (index(s, "-pre.")) { pre = substr(s, index(s, "-pre.") + 5); s = substr(s, 1, index(s, "-pre.") - 1); rank = 0 } - split(s, v, ".") - printf "%d %d %d %d %d\n", v[1], v[2], v[3], rank, (pre == "" ? 0 : pre) - }' -} - -newer=$( - gh api --paginate "repos/$repo/git/matching-refs/tags/v" --jq '.[].ref' | - sed 's#^refs/tags/##' | - { grep -E '^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$' || true; } | - while read -r t; do echo "$(key "$t") $t"; done | - awk -v p="$(key "$pin")" ' - BEGIN { split(p, k, " ") } - { - for (i = 1; i <= 5; i++) { - if ($i + 0 > k[i] + 0) { print; next } - if ($i + 0 < k[i] + 0) next - } - }' | - sort -n -k1,1 -k2,2 -k3,3 -k4,4 -k5,5 | - awk '{ print $6 }' -) - -[ -n "$newer" ] || exit 0 -if [ "$all" -eq 1 ]; then - echo "$newer" - exit 0 -fi - -# Stable tags, then the last tag if it is a pre-release (newer than every stable). -grep -v -- '-pre\.' <<<"$newer" || true -last=$(tail -1 <<<"$newer") -case $last in *-pre.*) echo "$last" ;; esac diff --git a/scripts/core-sync/drift-check.sh b/scripts/core-sync/drift-check.sh deleted file mode 100755 index 78fb4e6..0000000 --- a/scripts/core-sync/drift-check.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -# drift-check.sh: compare core main with the pinned core tag. -# -# Usage: drift-check.sh --out -# -# Run from the root of an sdk-python clone. Three checks: -# 1. buf breaking: core main against the pinned tag (FILE rules), for the -# proto files the SDK generates (what `mise run proto:fetch` exports) -# 2. regenerate and test: bindings from core main in a temporary -# worktree, then the unit tests -# 3. uncovered RPCs: methods in the regenerated *_pb2_grpc.py stubs that no -# SDK code under src/openshell calls (the bindings excluded); -# informational, it does not fail the check -# Writes a markdown report with the header -# "Pin: | Core main: | Result: pass|fail" and one section per -# check. Exits 0 when checks 1 and 2 pass, 1 otherwise. -set -euo pipefail - -core_url=${CORE_URL:-https://github.com/NVIDIA/OpenShell.git} -out= -while [ $# -gt 0 ]; do - case $1 in - --out) out=${2:?--out needs a file}; shift 2 ;; - *) echo "usage: $0 --out " >&2; exit 2 ;; - esac -done -[ -n "$out" ] || { echo "usage: $0 --out " >&2; exit 2; } -[ "$(git rev-parse --show-toplevel)" = "$(pwd -P)" ] || { - echo "ERROR: run this from the root of the sdk-python clone" >&2 - exit 1 -} -case $out in /*) ;; *) out="$(pwd)/$out" ;; esac - -pin_of() { sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p'; } -pin=$(pin_of &2; exit 1; } -[[ "$pin" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || { - echo "ERROR: pin '$pin' is not a release tag; commit SHA pins are not supported by core-sync" >&2 - exit 1 -} -main_sha=$(git ls-remote "$core_url" refs/heads/main | cut -f1) -[ -n "$main_sha" ] || { echo "ERROR: cannot resolve core main" >&2; exit 1; } - -work=$(mktemp -d) -cleanup() { git worktree remove --force "$work/wt" >/dev/null 2>&1 || true; rm -rf "$work"; } -trap cleanup EXIT -result=pass -fence() { printf '```text\n%s\n```\n' "$1"; } - -# 1. buf breaking for the files the SDK generates: the two roots of -# proto:fetch and their imports, as exported at the pin. -buf export "${core_url}#ref=${pin},depth=1,subdir=proto" \ - --path openshell.proto --path credential_driver.proto --output "$work/pinned" -files=$(cd "$work/pinned" && find . -name '*.proto' | sed 's#^\./##' | sort) -path_args=() -for f in $files; do path_args+=(--path "proto/$f"); done -# Core's root buf.yaml has two modules; this config keeps only proto/, so -# its imports resolve the same way buf export sees them. -if breaking=$(buf breaking "${core_url}#ref=${main_sha}" \ - --against "${core_url}#tag=${pin}" \ - --config '{"version":"v2","modules":[{"path":"proto"}],"breaking":{"use":["FILE"]}}' \ - "${path_args[@]}" 2>&1); then - breaking_section="No breaking changes in $(echo $files | sed 's/ /, /g') between \`$pin\` and main." -else - result=fail - breaking_section="Breaking changes between \`$pin\` and main: - -$(fence "$breaking")" -fi - -# 2. Regenerate from main and run the unit tests. -git worktree add --quiet --detach "$work/wt" HEAD -if build=$( - cd "$work/wt" && - sed -i.bak "s/^OPENSHELL_CORE_REF = \".*\"$/OPENSHELL_CORE_REF = \"$main_sha\"/" mise.toml && - rm -f mise.toml.bak && - mise trust --quiet mise.toml && - mise run --quiet proto:gen 2>&1 && - mise run --quiet test 2>&1 -); then - changed=$(git -C "$work/wt" diff --stat -- src/openshell/_proto | tail -1 | sed 's/^ *//') - build_section="Bindings from main pass the unit tests (${changed:-no generated files change})." -else - result=fail - build_section="Regenerating from main or testing against it fails: - -$(fence "$(tail -n 60 <<<"$build")")" -fi - -# 3. RPCs the SDK does not call yet. Full method names look like -# "/openshell.v1.OpenShell/CreateSandbox". -uncovered="" -if ls "$work/wt"/src/openshell/_proto/*_pb2_grpc.py >/dev/null 2>&1; then - methods=$(grep -ho "['\"]/[a-zA-Z0-9_.]*\.[A-Za-z0-9_]*/[A-Za-z0-9_]*['\"]" \ - "$work/wt"/src/openshell/_proto/*_pb2_grpc.py | tr -d "'\"" | sort -u) - for m in $methods; do - service=${m#/}; service=${service%%/*}; service=${service##*.} - method=${m##*/} - if ! grep -rqE "\.${method}\(" src/openshell --include='*.py' --exclude-dir=_proto; then - uncovered+="- ${service}.${method}"$'\n' - fi - done -fi -if [ -z "$uncovered" ]; then - uncovered_section="Every RPC in the regenerated stubs is called by the SDK." -else - uncovered_section="RPCs in core main that the SDK does not call yet ($(grep -c . <<<"$uncovered")): - -${uncovered%$'\n'}" -fi - -cat >"$out" < " lines of a body's failures section. -failure_lines() { - awk -v h="$failures_heading" ' - $0 == h { f = 1; next } - f && /^## / { f = 0 } - f && /^- / { print }' <<<"$1" -} - -# compose_body -compose_body() { - printf '%s\n\n%s\n\n' "$1" "$failures_heading" - if [ -n "$2" ]; then - printf '%s\n\n' "$2" - else - printf 'None.\n\n' - fi - printf '%s\n' "Automation appends a dated line here when core-watch or core-drift fails. Delete a line once it is handled; the issue closes when the drift check passes and no lines are left." -} diff --git a/scripts/core-sync/open-bump-pr.sh b/scripts/core-sync/open-bump-pr.sh deleted file mode 100755 index f73f183..0000000 --- a/scripts/core-sync/open-bump-pr.sh +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -# open-bump-pr.sh: open the update proposal that moves the core pin to . -# -# Usage: open-bump-pr.sh [--dry-run] [--base ] -# -# Run from the root of an sdk-python clone. Builds branch core/ from -# origin/main (or --base) in a temporary worktree: pin change -# (OPENSHELL_CORE_REF in mise.toml), regenerated bindings, refreshed copies of -# core files (scripts/vendored-core-files.txt), one commit -# "chore(core): track ". Then pushes it and opens a PR labeled -# core-bump plus stable or prerelease, and closes open pre-release proposals -# for older tags. --dry-run does the local part, prints the PR body, and -# changes nothing on GitHub. -# -# Skips the tag when any PR from branch core/ exists (open, merged, or -# closed), so a closed proposal is never reopened. A branch on origin without -# a PR is the leftover of a run that pushed but failed to open the PR; the -# run rebuilds it and replaces the branch (force-with-lease) so the PR gets -# opened. When the core tag no longer exists, closes an open proposal for it. -# The current pin must be a release tag; a commit SHA pin stops the run. -# Git identity and GH_TOKEN come from the environment (the App in CI). -set -euo pipefail - -core_repo=${CORE_REPO:-NVIDIA/OpenShell} -sdk_repo=${SDK_REPO:-ai-openshell/sdk-python} -image=ghcr.io/nvidia/openshell/gateway -diff_cap=400 - -tag= -dry=0 -base=origin/main -while [ $# -gt 0 ]; do - case $1 in - --dry-run) dry=1; shift ;; - --base) base=${2:?--base needs a ref}; shift 2 ;; - -*) echo "usage: $0 [--dry-run] [--base ]" >&2; exit 2 ;; - *) [ -z "$tag" ] || { echo "usage: $0 [--dry-run]" >&2; exit 2; }; tag=$1; shift ;; - esac -done -[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || { - echo "ERROR: '$tag' is not a core release tag (vX.Y.Z or vX.Y.Z-pre.N)" >&2 - exit 2 -} - -here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -body_tmpl="$here/bump-pr-body.md" -branch="core/$tag" -case $tag in *-pre.*) kind=prerelease ;; *) kind=stable ;; esac - -die() { echo "ERROR: $*" >&2; exit 1; } - -# older : true when release tag a sorts below b (vX.Y.Z-pre.N < vX.Y.Z). -older() { - awk -v a="$1" -v b="$2" ' - function key(t, s, pre, rank, v) { - s = t; sub(/^v/, "", s); pre = 0; rank = 1 - if (index(s, "-pre.")) { pre = substr(s, index(s, "-pre.") + 5) + 0; s = substr(s, 1, index(s, "-pre.") - 1); rank = 0 } - split(s, v, ".") - return sprintf("%06d%06d%06d%d%06d", v[1], v[2], v[3], rank, pre) - } - BEGIN { exit !(key(a) < key(b)) }' -} -run() { if [ "$dry" -eq 1 ]; then echo "dry-run: $*"; else "$@"; fi; } -pin_of() { sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p'; } - -[ "$(git rev-parse --show-toplevel)" = "$(pwd -P)" ] || die "run this from the root of the sdk-python clone" -[ -f "$body_tmpl" ] || die "$body_tmpl is missing" - -open_pr_for_branch() { - gh pr list -R "$sdk_repo" --head "$branch" --state open --json number --jq '.[0].number // empty' -} - -# The core tag may have been deleted (a withdrawn pre-release). Only a 404 -# means that; any other failure (rate limit, 5xx) stops the run. -if ! err=$(gh api "repos/$core_repo/git/ref/tags/$tag" --silent 2>&1); then - grep -q 'HTTP 404' <<<"$err" || die "looking up core tag $tag failed: $err" - n=$(open_pr_for_branch) - if [ -n "$n" ]; then - echo "Core tag $tag is gone; closing #$n." - run gh pr close "$n" -R "$sdk_repo" --delete-branch \ - --comment "Core tag \`$tag\` no longer exists in $core_repo, so this proposal is closed." - else - echo "Core tag $tag does not exist; nothing to do." - fi - exit 0 -fi - -# Any earlier proposal is final: merged, closed by a maintainer, or closed as -# superseded by a newer pre-release. Its branch may be gone, so ask for PRs. -prev=$(gh pr list -R "$sdk_repo" --head "$branch" --state all --json number,state \ - --jq '.[0] | select(.) | "#\(.number) (\(.state | ascii_downcase))"') -if [ -n "$prev" ]; then - echo "Proposal $prev exists for $tag already; skipping." - exit 0 -fi -# No PR, but the branch may exist: an earlier run pushed it and then failed -# to open the PR. Remember its tip so the push replaces exactly that. -stale_tip=$(git ls-remote --heads origin "$branch" | cut -f1) -[ -z "$stale_tip" ] || echo "Branch $branch exists on origin without a PR; rebuilding it for $tag." - -git fetch --quiet origin -git rev-parse --verify --quiet "$base^{commit}" >/dev/null || die "base $base is not a commit" -old=$(git show "$base:mise.toml" | pin_of) -[ -n "$old" ] || die "no OPENSHELL_CORE_REF in mise.toml on $base" -[[ "$old" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || - die "OPENSHELL_CORE_REF on $base is '$old', not a release tag; the core-sync automation does not support commit SHA pins" -[ "$old" != "$tag" ] || { echo "The pin on $base is $tag already; skipping."; exit 0; } - -work=$(mktemp -d) -cleanup() { git worktree remove --force "$work/wt" >/dev/null 2>&1 || true; rm -rf "$work"; } -trap cleanup EXIT -git worktree add --quiet -b "$branch" "$work/wt" "$base" - -( - cd "$work/wt" - sed -i.bak "s/^OPENSHELL_CORE_REF = \".*\"$/OPENSHELL_CORE_REF = \"$tag\"/" mise.toml && rm -f mise.toml.bak - [ "$(pin_of &2; exit 1; } - mise trust --quiet mise.toml - log=$(mise run --quiet proto:gen 2>&1) || { echo "$log" >&2; exit 1; } - - # Refresh the copies of core files; one not in the tag yet is left as is. - : >"$work/fixtures.txt" - while read -r src local; do - status=$(curl -sS -o "$work/fixture" -w '%{http_code}' \ - "https://raw.githubusercontent.com/$core_repo/$tag/$src") - case $status in - 200) - if ! cmp -s "$work/fixture" "$local"; then - cp "$work/fixture" "$local" - echo "- \`$local\` updated from core \`$tag\`" >>"$work/fixtures.txt" - fi - ;; - 404) echo "- \`$local\` is not in core \`$tag\` yet, left unchanged" >>"$work/fixtures.txt" ;; - *) echo "ERROR: fetching $src at $tag returned HTTP $status" >&2; exit 1 ;; - esac - done < <(grep -v '^#' scripts/vendored-core-files.txt) - - # Keep the README's "currently vX.Y.Z" in step with stable pins. - if [ "$kind" = stable ]; then - sed -i.bak "s/currently \`${old}\`/currently \`${tag}\`/" README.md && rm -f README.md.bak - fi - - git add -A - git commit --quiet --signoff -m "chore(core): track $tag" -m "Move the core pin from $old to $tag and regenerate the bindings." -) - -# Proto diff between the two core tags. The compare API caps its file list at -# 300 entries, so fetch both tags shallow and blobless and diff locally; only -# the proto/ blobs are downloaded. -compare_url="https://github.com/$core_repo/compare/$old...$tag" -git init --quiet --bare "$work/core.git" -git -C "$work/core.git" fetch --quiet --depth=1 --filter=blob:none --no-tags \ - "https://github.com/$core_repo.git" "refs/tags/$old:refs/tags/$old" "refs/tags/$tag:refs/tags/$tag" -proto_diff=$(git -C "$work/core.git" diff "$old" "$tag" -- proto/) -if [ -z "$proto_diff" ]; then - proto_section="No changes under \`proto/\` between \`$old\` and \`$tag\`." -else - lines=$(wc -l <<<"$proto_diff" | tr -d ' ') - shown=$(head -n "$diff_cap" <<<"$proto_diff") - proto_section=$(printf '```diff\n%s\n```' "$shown") - if [ "$lines" -gt "$diff_cap" ]; then - proto_section=$(printf '%s\n\nShowing %s of %s lines; see the [full compare](%s).' "$proto_section" "$diff_cap" "$lines" "$compare_url") - fi -fi -generated=$(git diff --stat "$base" "$branch" -- src/openshell/_proto | tail -1 | sed "s/^ *//") -fixtures=$(cat "$work/fixtures.txt") - -body="$work/body.md" -OLD="$old" NEW="$tag" KIND="$kind" COMPARE_URL="$compare_url" IMAGE="$image:${tag#v}" \ -GENERATED="${generated:-no generated files changed}" FIXTURES="${fixtures:- - none changed}" \ -PROTO_DIFF="$proto_section" \ - awk '{ - while (match($0, /\{\{[A-Z_]+\}\}/)) { - key = substr($0, RSTART + 2, RLENGTH - 4) - $0 = substr($0, 1, RSTART - 1) ENVIRON[key] substr($0, RSTART + RLENGTH) - } - print - }' "$body_tmpl" >"$body" - -echo "Proposal for $tag ($kind), pin $old -> $tag:" -git log --format=' %h %an <%ae> %s' "$base..$branch" -git diff --stat "$base" "$branch" | tail -1 | sed 's/^/ /' - -if [ "$dry" -eq 1 ]; then - echo - echo "PR title: chore(core): track $tag" - echo "PR body:" - sed 's/^/ | /' "$body" - cleanup - git branch -D "$branch" >/dev/null - echo - echo "dry-run: nothing pushed; local branch $branch removed." - exit 0 -fi - -if [ -n "$stale_tip" ]; then - git push --quiet --force-with-lease="$branch:$stale_tip" origin "$branch" -else - git push --quiet origin "$branch" -fi -url=$(gh pr create -R "$sdk_repo" --base main --head "$branch" \ - --title "chore(core): track $tag" --body-file "$body" \ - --label core-bump --label "$kind") -echo "Opened $url" -new=${url##*/} - -# Close open pre-release proposals for older tags; this one supersedes them. -gh pr list -R "$sdk_repo" --label core-bump --label prerelease --state open \ - --json number,headRefName --jq '.[] | "\(.number) \(.headRefName)"' | - while read -r n head; do - other=${head#core/} - [ "$n" != "$new" ] && [ "$other" != "$tag" ] || continue - if older "$other" "$tag"; then - echo "Closing #$n ($other), superseded by #$new." - gh pr close "$n" -R "$sdk_repo" --delete-branch \ - --comment "Superseded by #$new, which tracks core \`$tag\`." - fi - done diff --git a/scripts/core-sync/report-failure.sh b/scripts/core-sync/report-failure.sh deleted file mode 100755 index 7e5e29a..0000000 --- a/scripts/core-sync/report-failure.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -# report-failure.sh: record a failed automation run in the drift issue. -# -# Usage: report-failure.sh [--dry-run] -# -# Appends "- " to the "## Automation failures" section -# of the open "Core main drift" issue, creating the issue when none is open. -# Also writes the line to the job summary. -set -euo pipefail -# shellcheck source=scripts/core-sync/drift-issue.sh -. "$(dirname "${BASH_SOURCE[0]}")/drift-issue.sh" - -dry=0 -args=() -for a in "$@"; do - case $a in --dry-run) dry=1 ;; *) args+=("$a") ;; esac -done -[ ${#args[@]} -eq 2 ] || { echo "usage: $0 [--dry-run]" >&2; exit 2; } -line="- $(date -u +%Y-%m-%d) ${args[0]} ${args[1]}" -[ -z "${GITHUB_STEP_SUMMARY:-}" ] || echo "Automation failure: $line" >>"$GITHUB_STEP_SUMMARY" - -n=$(drift_issue_number) -if [ -n "$n" ]; then - current=$(gh issue view "$n" -R "$sdk_repo" --json body --jq .body) - report=$(awk -v h="$failures_heading" '$0 == h { exit } { print }' <<<"$current" | sed -e :a -e '/^\n*$/{$d;N;ba' -e '}') - failures=$(failure_lines "$current") -else - report="No drift report yet; the next core-drift run fills this in." - failures= -fi -failures=$(printf '%s\n%s' "$failures" "$line" | sed '/^$/d') -body=$(compose_body "$report" "$failures") - -if [ "$dry" -eq 1 ]; then - if [ -n "$n" ]; then target="update #$n"; else target="create the issue"; fi - echo "dry-run: would $target with:" - sed 's/^/ | /' <<<"$body" - exit 0 -fi -if [ -n "$n" ]; then - gh issue edit "$n" -R "$sdk_repo" --body "$body" >/dev/null - echo "Recorded the failure in #$n." -else - gh issue create -R "$sdk_repo" --title "$drift_title" --label "$drift_label" --body "$body" -fi diff --git a/scripts/core-sync/update-drift-issue.sh b/scripts/core-sync/update-drift-issue.sh deleted file mode 100755 index c174bcb..0000000 --- a/scripts/core-sync/update-drift-issue.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -# update-drift-issue.sh: keep the single "Core main drift" issue current. -# -# Usage: update-drift-issue.sh [--dry-run] -# -# The report comes from drift-check.sh. The issue body is the report plus the -# "## Automation failures" section, which is carried over unchanged. With -# "Result: fail" or pending failures, the issue is created or its body -# replaced. With "Result: pass" and no pending failures, an open issue is -# closed with a comment, and nothing is created. -set -euo pipefail -# shellcheck source=scripts/core-sync/drift-issue.sh -. "$(dirname "${BASH_SOURCE[0]}")/drift-issue.sh" - -report= -dry=0 -for a in "$@"; do - case $a in - --dry-run) dry=1 ;; - -*) echo "usage: $0 [--dry-run]" >&2; exit 2 ;; - *) report=$a ;; - esac -done -[ -f "$report" ] || { echo "usage: $0 [--dry-run]" >&2; exit 2; } - -text=$(cat "$report") -header=$(head -1 <<<"$text") -case $header in -*"Result: pass"*) result=pass ;; -*"Result: fail"*) result=fail ;; -*) echo "ERROR: $report has no 'Result: pass|fail' header" >&2; exit 1 ;; -esac - -n=$(drift_issue_number) -failures= -[ -z "$n" ] || failures=$(failure_lines "$(gh issue view "$n" -R "$sdk_repo" --json body --jq .body)") -body=$(compose_body "$text" "$failures") - -if [ "$result" = pass ] && [ -z "$failures" ]; then - if [ -z "$n" ]; then - echo "Drift check passes and no failures are pending; no issue needed." - exit 0 - fi - echo "Closing #$n: the drift check passes and no failures are pending." - [ "$dry" -eq 1 ] && exit 0 - gh issue edit "$n" -R "$sdk_repo" --body "$body" >/dev/null - gh issue close "$n" -R "$sdk_repo" --comment "The drift check passes against core main ($(sed 's/.*Core main: \([0-9a-f]*\).*/\1/' <<<"$header")) and no automation failures are pending." - exit 0 -fi - -if [ "$dry" -eq 1 ]; then - if [ -n "$n" ]; then target="update #$n"; else target="create the issue"; fi - echo "dry-run: would $target with:" - sed 's/^/ | /' <<<"$body" - exit 0 -fi -if [ -n "$n" ]; then - gh issue edit "$n" -R "$sdk_repo" --body "$body" >/dev/null - echo "Updated #$n." -else - gh issue create -R "$sdk_repo" --title "$drift_title" --label "$drift_label" --body "$body" -fi diff --git a/scripts/release/tag-release.sh b/scripts/release/tag-release.sh deleted file mode 100755 index 2cf45f3..0000000 --- a/scripts/release/tag-release.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -# tag-release.sh: start the lockstep SDK release for a stable core pin. -# -# Usage: tag-release.sh [--dry-run] [--force] [--sha ] [--since ] -# -# Reads the pin, OPENSHELL_CORE_REF in mise.toml, at (default: -# $GITHUB_SHA, else HEAD). A pre-release pin creates nothing. Without --force, -# nothing is created either when the pin at equals the pin at --since -# (default: the parent of ) or there is no pin there, so ordinary -# pushes to main and the push that adds mise.toml are no-ops. -# The release workflow passes the commit before the push as --since, which -# covers pushes of several commits. Then, each step only when missing: -# - tag vX.Y.Z (lightweight) at ; a tag at another commit fails -# - a draft GitHub release with the merged PR's body plus -# "Matches OpenShell core vX.Y.Z" -# Prints "tag=vX.Y.Z" when the release still has to be built and published, -# and nothing when there is nothing to release or the release is published -# already. The workflow appends the output to $GITHUB_OUTPUT. -set -euo pipefail - -sdk_repo=${SDK_REPO:-ai-openshell/sdk-python} -dry=0 -force=0 -sha=${GITHUB_SHA:-} -since= - -usage() { echo "usage: $0 [--dry-run] [--force] [--sha ] [--since ]" >&2; exit 2; } -while [ $# -gt 0 ]; do - case $1 in - --dry-run) dry=1; shift ;; - --force) force=1; shift ;; - --sha) sha=${2:?--sha needs a commit}; shift 2 ;; - --since) since=${2:?--since needs a commit}; shift 2 ;; - *) usage ;; - esac -done -[ -n "$sha" ] || sha=$(git rev-parse HEAD) -sha=$(git rev-parse "$sha^{commit}") -short=$(git rev-parse --short "$sha") - -pin_at() { git show "$1:mise.toml" 2>/dev/null | sed -n 's/^OPENSHELL_CORE_REF = "\(.*\)"$/\1/p'; } - -pin=$(pin_at "$sha") -if ! [[ "$pin" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Pin $pin is not a stable release; no SDK release." >&2 - exit 0 -fi - -if [ "$force" -eq 0 ]; then - # A new branch push reports an all-zero "before"; fall back to the parent. - if [ -z "$since" ] || ! git rev-parse --verify --quiet "$since^{commit}" >/dev/null; then - since="$sha^" - fi - old=$(pin_at "$since" || true) - if [ "$old" = "$pin" ]; then - echo "The pin is $pin both at $short and at $(git rev-parse --short "$since"); no SDK release." >&2 - exit 0 - fi - # No pin before means the push added mise.toml (the import), not a bump. - if [ -z "$old" ]; then - echo "No pin at $(git rev-parse --short "$since"); no SDK release." >&2 - exit 0 - fi -fi - -# get_or_empty : the value, empty on 404, and fail on any other -# error. gh prints the error body on stdout for a 404, so stdout is only used -# after a successful call. -get_or_empty() { - local out err - if out=$(gh api "$1" --jq "$2" 2>"${tmp}/err"); then - printf '%s' "$out" - else - err=$(cat "${tmp}/err") - grep -q 'HTTP 404' <<<"$err" || { echo "ERROR: gh api $1 failed: $err" >&2; return 1; } - fi -} -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT - -existing=$(get_or_empty "repos/$sdk_repo/git/ref/tags/$pin" '.object.sha') -if [ -n "$existing" ] && [ "$existing" != "$sha" ]; then - echo "ERROR: tag $pin exists at $existing, not at $sha. SDK tags never move; fix this by hand." >&2 - exit 1 -fi -# The tag lookup doesn't return drafts, so search the release list. -release=$(gh api --paginate "repos/$sdk_repo/releases" \ - --jq ".[] | select(.tag_name == \"$pin\") | if .draft then \"draft\" else \"published\" end" | head -1) -if [ "$release" = published ]; then - echo "Release $pin is published already; nothing to do." >&2 - exit 0 -fi - -# Release notes: the body of the PR that merged this commit, then the match line. -notes="$tmp/notes.md" -pr_body=$(get_or_empty "repos/$sdk_repo/commits/$sha/pulls" '.[0].body // empty') -{ - [ -z "$pr_body" ] || printf '%s\n\n' "$pr_body" - echo "Matches OpenShell core $pin." -} >"$notes" - -if [ "$dry" -eq 1 ]; then - if [ -n "$existing" ]; then - echo "dry-run: tag $pin exists at $sha" >&2 - else - echo "dry-run: would create tag $pin at $sha" >&2 - fi - if [ "$release" = draft ]; then - echo "dry-run: draft release $pin exists" >&2 - else - echo "dry-run: would create draft release $pin with notes:" >&2 - sed 's/^/ | /' "$notes" >&2 - fi - echo "tag=$pin" - exit 0 -fi - -if [ -z "$existing" ]; then - gh api -X POST "repos/$sdk_repo/git/refs" -f ref="refs/tags/$pin" -f sha="$sha" --silent - echo "Created tag $pin at $sha." >&2 -fi -if [ -z "$release" ]; then - gh release create "$pin" -R "$sdk_repo" --draft --verify-tag --title "$pin" --notes-file "$notes" >&2 -fi -echo "tag=$pin" diff --git a/scripts/uncovered-rpcs.sh b/scripts/uncovered-rpcs.sh new file mode 100755 index 0000000..73fab90 --- /dev/null +++ b/scripts/uncovered-rpcs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# uncovered-rpcs.sh: print the RPCs in the generated gRPC stubs +# (src/openshell/_proto/*_pb2_grpc.py) that no SDK code under src/openshell +# calls, one "Service.Method" per line. The bindings themselves don't count. +# Full method names in the stubs look like "/openshell.v1.OpenShell/CreateSandbox". +set -euo pipefail + +ls src/openshell/_proto/*_pb2_grpc.py >/dev/null 2>&1 || exit 0 +grep -ho "['\"]/[a-zA-Z0-9_.]*\.[A-Za-z0-9_]*/[A-Za-z0-9_]*['\"]" \ + src/openshell/_proto/*_pb2_grpc.py | tr -d "'\"" | sort -u | + while read -r m; do + service=${m#/}; service=${service%%/*}; service=${service##*.} + method=${m##*/} + grep -rqE "\.${method}\(" src/openshell --include='*.py' --exclude-dir=_proto || + echo "${service}.${method}" + done diff --git a/scripts/vendored-core-files.txt b/scripts/vendored-core-files.txt index ba6e08a..0351bfc 100644 --- a/scripts/vendored-core-files.txt +++ b/scripts/vendored-core-files.txt @@ -2,7 +2,7 @@ # # Files copied from NVIDIA/OpenShell at the pinned ref, one per line: # . `mise run vendored:check` fails when a copy -# differs from core, and core bump PRs refresh them (core-sync/open-bump-pr.sh). +# differs from core, and core bump PRs refresh them (mise run core:bump). sdk/conformance/oauth-client-credentials.json tests/fixtures/oauth-client-credentials.json sdk/conformance/oauth-client-credentials.json.license tests/fixtures/oauth-client-credentials.json.license e2e/python/Dockerfile.workload tests/e2e/Dockerfile.workload From 735d08950e3fbbb7e1bc3fe220e6cd1fefe901de Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 19 Nov 2026 10:45:19 -0800 Subject: [PATCH 2/3] ci(core-sync): pin the shared scripts through the workflow ref only The shared workflows now run the scripts of their own commit, so the mise remote tasks and their second ref go. scripts/core-sync.sh runs the scripts from CORE_SYNC_DIR in CI and, locally, from the commit that core-drift.yml calls. The release tag job calls the shared release-tag.yml at the same commit, and Dependabot groups the three workflow refs. Signed-off-by: Piotr Mlocek --- .github/dependabot.yml | 4 ++++ .github/workflows/core-drift.yml | 6 ++--- .github/workflows/core-watch.yml | 5 ++-- .github/workflows/release.yml | 41 ++++---------------------------- mise.toml | 20 ++++++++-------- scripts/README.md | 4 ++-- scripts/core-sync.sh | 35 +++++++++++++++++++++++++++ 7 files changed, 60 insertions(+), 55 deletions(-) create mode 100755 scripts/core-sync.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d05c47..dd8e49d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,3 +29,7 @@ updates: codeql: patterns: - "github/codeql-action*" + # The shared core-sync workflows, all called at one commit. + core-sync: + patterns: + - "ai-openshell/workflows*" diff --git a/.github/workflows/core-drift.yml b/.github/workflows/core-drift.yml index 3d863b7..0bfc003 100644 --- a/.github/workflows/core-drift.yml +++ b/.github/workflows/core-drift.yml @@ -2,8 +2,8 @@ # Nightly comparison of core main with the pinned core tag; keeps the single # "Core main drift" issue current (mise run core:drift). Uses the shared -# workflow from ai-openshell/workflows (core-sync/). Keep the ref in step with -# the core-sync tasks in mise.toml. +# workflow from ai-openshell/workflows (core-sync/). Its commit is the pin for +# the core-sync scripts too: scripts/core-sync.sh reads it from this line. name: core-drift on: @@ -16,4 +16,4 @@ permissions: jobs: drift: - uses: pimlock/workflows/.github/workflows/core-drift.yml@63d4b041b9b99fdfa3491573d31a537451a427ab # feat/core-sync/pimlock + uses: pimlock/workflows/.github/workflows/core-drift.yml@33ae68eb87030b53b479f4ed8426f458c1d52b2d # feat/core-sync/pimlock diff --git a/.github/workflows/core-watch.yml b/.github/workflows/core-watch.yml index 9f71cc6..3279168 100644 --- a/.github/workflows/core-watch.yml +++ b/.github/workflows/core-watch.yml @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Opens one update proposal per new core release tag (mise run core:bump), -# with the shared workflow from ai-openshell/workflows (core-sync/). Keep the -# ref in step with the core-sync tasks in mise.toml. +# with the shared workflow from ai-openshell/workflows (core-sync/). name: core-watch on: @@ -20,6 +19,6 @@ permissions: jobs: watch: - uses: pimlock/workflows/.github/workflows/core-watch.yml@63d4b041b9b99fdfa3491573d31a537451a427ab # feat/core-sync/pimlock + uses: pimlock/workflows/.github/workflows/core-watch.yml@33ae68eb87030b53b479f4ed8426f458c1d52b2d # feat/core-sync/pimlock with: tag: ${{ inputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81cff88..1c3bc83 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,46 +28,13 @@ jobs: name: tag # Only main releases, also when the workflow is started by hand. if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - timeout-minutes: 10 + # Shared with the other SDKs; same commit as core-watch.yml and core-drift.yml. + uses: pimlock/workflows/.github/workflows/release-tag.yml@33ae68eb87030b53b479f4ed8426f458c1d52b2d # feat/core-sync/pimlock permissions: contents: write # create the tag and the draft release pull-requests: read # read the merged PR's body for the release notes - outputs: - tag: ${{ steps.tag.outputs.tag }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - # Only for the shared release:tag task (git and gh); no tools needed. - - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 - with: - install: false - cache: false - - - id: tag - name: Tag the release and create a draft - env: - GH_TOKEN: ${{ github.token }} - EVENT: ${{ github.event_name }} - INPUT_SHA: ${{ inputs.sha }} - BEFORE: ${{ github.event.before }} - run: | - set -euo pipefail - if [ "$EVENT" = workflow_dispatch ]; then - sha=${INPUT_SHA:-$GITHUB_SHA} - git merge-base --is-ancestor "$sha" "$GITHUB_SHA" || { - echo "::error::$sha is not on main" - exit 1 - } - mise run --quiet release:tag -- --force --sha "$sha" >>"$GITHUB_OUTPUT" - else - # Compare against the commit before the push, so a pin change - # that isn't the pushed tip still counts. - mise run --quiet release:tag -- --since "$BEFORE" >>"$GITHUB_OUTPUT" - fi + with: + sha: ${{ inputs.sha }} build: name: build diff --git a/mise.toml b/mise.toml index 4495552..6c72df3 100644 --- a/mise.toml +++ b/mise.toml @@ -150,34 +150,34 @@ description = "Run the full CI pipeline" depends = ["check", "test", "build"] # --- Core sync -------------------------------------------------------------- -# The scripts are shared through ai-openshell/workflows (core-sync/), pinned to -# the same commit as the reusable workflows in core-watch.yml and -# core-drift.yml; update the refs together. They call the tasks below this -# block (core-sync/README.md there has the contract). +# The scripts are shared through ai-openshell/workflows (core-sync/). +# scripts/core-sync.sh runs them at the commit the workflows call (the `uses:` +# line in core-drift.yml is the pin). They call the tasks below this block +# (core-sync/README.md there has the contract). [tasks."core:tags"] description = "List core release tags newer than the pin (newest pre-release only, --all for every tag)" -file = "git::https://github.com/pimlock/workflows.git//core-sync/core-tags.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" +run = "scripts/core-sync.sh core-tags.sh" [tasks."core:bump"] description = "Open the update proposal for a core tag: mise run core:bump -- [--dry-run]" -file = "git::https://github.com/pimlock/workflows.git//core-sync/open-bump-pr.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" +run = "scripts/core-sync.sh open-bump-pr.sh" [tasks."core:drift"] description = "Compare core main with the pin: mise run core:drift -- --out " -file = "git::https://github.com/pimlock/workflows.git//core-sync/drift-check.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" +run = "scripts/core-sync.sh drift-check.sh" [tasks."core:drift-issue"] description = "Create, update, or close the core main drift issue from a report: mise run core:drift-issue -- [--dry-run]" -file = "git::https://github.com/pimlock/workflows.git//core-sync/update-drift-issue.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" +run = "scripts/core-sync.sh update-drift-issue.sh" [tasks."core:report-failure"] description = "Record a failed automation run in the drift issue: mise run core:report-failure -- [--dry-run]" -file = "git::https://github.com/pimlock/workflows.git//core-sync/report-failure.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" +run = "scripts/core-sync.sh report-failure.sh" [tasks."release:tag"] description = "Tag and draft the release when a stable pin lands on main: mise run release:tag -- [--dry-run] [--force] [--sha ] [--since ]" -file = "git::https://github.com/pimlock/workflows.git//core-sync/tag-release.sh?ref=63d4b041b9b99fdfa3491573d31a537451a427ab" +run = "scripts/core-sync.sh tag-release.sh" [tasks."proto:buf"] description = "Run buf: mise run proto:buf -- " diff --git a/scripts/README.md b/scripts/README.md index 870c7d2..b5866f8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,11 +4,11 @@ The workflows under `.github/workflows/` run these scripts, most through a mise ## Core sync and release tagging (shared) -The core-sync scripts and the `core-watch` and `core-drift` workflows are shared with the other SDKs through [ai-openshell/workflows](https://github.com/ai-openshell/workflows/tree/main/core-sync), which documents them and the contract this repository implements. `mise.toml` pins the scripts as remote tasks, and `core-watch.yml` and `core-drift.yml` call the shared workflows at the same commit; update the two refs together. +The core-sync scripts and the `core-watch` and `core-drift` workflows are shared with the other SDKs through [ai-openshell/workflows](https://github.com/ai-openshell/workflows/tree/main/core-sync), which documents them and the contract this repository implements. The workflows (`core-watch.yml`, `core-drift.yml`, and the tag job of `release.yml`) call the shared workflows at one commit, and the shared workflows run that commit's scripts. Locally, `core-sync.sh` runs the scripts of the commit in `core-drift.yml`'s `uses:` line, downloaded once into `~/.cache/openshell-core-sync/`. | mise task | Runs | |-----------|------| -| `core:tags`, `core:bump`, `core:drift`, `core:drift-issue`, `core:report-failure`, `release:tag` | The shared scripts. | +| `core:tags`, `core:bump`, `core:drift`, `core:drift-issue`, `core:report-failure`, `release:tag` | The shared scripts, through `core-sync.sh`. | | `proto:fetch`, `proto:gen`, `proto:buf`, `check`, `test` | This repository's parts of the contract. | | `core:uncovered-rpcs` | `uncovered-rpcs.sh`: RPCs in `src/openshell/_proto` the SDK doesn't call, for the drift report. | diff --git a/scripts/core-sync.sh b/scripts/core-sync.sh new file mode 100755 index 0000000..5c7505a --- /dev/null +++ b/scripts/core-sync.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# core-sync.sh: run a shared core-sync script from ai-openshell/workflows. +# SDK repositories keep a copy of this file as scripts/core-sync.sh and point +# their core-sync mise tasks at it. +# +# Usage: scripts/core-sync.sh