diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4bbf5db..b2b0bf9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,5 @@ # The sdk team owns the SDK; OpenShell maintainers review as a fallback. * @ai-openshell/sdk @ai-openshell/maintainers -# CI and the scripts it runs with the bot token or PyPI access need an infra review. +# CI and what it runs with the bot token or PyPI access need an infra review. /.github/ @ai-openshell/infra -/scripts/core-sync/ @ai-openshell/infra -/scripts/release/ @ai-openshell/infra diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d05c47..dd8e49d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,3 +29,7 @@ updates: codeql: patterns: - "github/codeql-action*" + # The shared core-sync workflows, all called at one commit. + core-sync: + patterns: + - "ai-openshell/workflows*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dcee41..77bb6cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,13 @@ jobs: # fetch-core.sh reads proto/ and sdk/conformance from NVIDIA/OpenShell # at the pinned ref (public, no credentials). + # The shared core-sync workflows run with the bot token: their pin must + # be one commit on ai-openshell/workflows main (not a fork-only commit). + - name: Check the core-sync pin + env: + GH_TOKEN: ${{ github.token }} + run: scripts/core-sync.sh --check-pin + - name: Format, lint, typecheck, proto and fixture drift run: mise run check diff --git a/.github/workflows/core-drift.yml b/.github/workflows/core-drift.yml index 94fc274..85cf03e 100644 --- a/.github/workflows/core-drift.yml +++ b/.github/workflows/core-drift.yml @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # Nightly comparison of core main with the pinned core tag; keeps the single -# "Core main drift" issue current (mise run core:drift). +# "Core main drift" issue current (mise run core:drift). Uses the shared +# workflow from ai-openshell/workflows (core-sync/). Its commit is the pin for +# the core-sync scripts too: scripts/core-sync.sh reads it from this line. name: core-drift on: @@ -12,48 +14,6 @@ on: permissions: contents: read -concurrency: - group: core-drift - cancel-in-progress: false - jobs: drift: - name: drift - # The bot secrets live in the sdk-bot environment, which only main can - # deploy to, so a pushed branch can't mint a bot token. - environment: sdk-bot - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - id: app-token - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 - with: - client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} - private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} - permission-issues: write - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 - - # Exit 1 with a report means drift; the issue records it. No report - # means the check itself broke, which the failure step records. - - name: Check drift against core main - run: | - mise run core:drift -- --out drift-report.md || true - test -s drift-report.md - cat drift-report.md >>"$GITHUB_STEP_SUMMARY" - - - name: Update the drift issue - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: mise run core:drift-issue -- drift-report.md - - - name: Report failure - if: failure() - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: mise run core:report-failure -- core-drift "$RUN_URL" + uses: pimlock/workflows/.github/workflows/core-drift.yml@2e0e58cd14e4a3484d0458a40466c77ef48b5f4f # feat/core-sync/pimlock diff --git a/.github/workflows/core-watch.yml b/.github/workflows/core-watch.yml index da79469..bb39386 100644 --- a/.github/workflows/core-watch.yml +++ b/.github/workflows/core-watch.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -# Opens one update proposal per new core release tag -# (mise run core:bump). +# Opens one update proposal per new core release tag (mise run core:bump), +# with the shared workflow from ai-openshell/workflows (core-sync/). name: core-watch on: @@ -17,68 +17,8 @@ on: permissions: contents: read -concurrency: - group: core-watch - cancel-in-progress: false - jobs: watch: - name: watch - # The bot secrets live in the sdk-bot environment, which only main can - # deploy to, so a pushed branch can't mint a bot token. - environment: sdk-bot - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - id: app-token - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 - with: - client-id: ${{ secrets.SDK_BOT_CLIENT_ID }} - private-key: ${{ secrets.SDK_BOT_PRIVATE_KEY }} - permission-contents: write - permission-pull-requests: write - permission-issues: write - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - token: ${{ steps.app-token.outputs.token }} - persist-credentials: true - - - uses: jdx/mise-action@b44d12e7d0d8ee7461bb893e1b1993500433185b # v5.2.7 - - - name: Set the bot identity - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} - run: | - set -euo pipefail - id=$(gh api "users/${APP_SLUG}[bot]" --jq .id) - git config user.name "${APP_SLUG}[bot]" - git config user.email "${id}+${APP_SLUG}[bot]@users.noreply.github.com" - - - name: Propose new core tags - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - INPUT_TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - if [ -n "$INPUT_TAG" ]; then - tags=$INPUT_TAG - else - tags=$(mise run --quiet core:tags) - fi - if [ -z "$tags" ]; then - echo "No core tags newer than $(mise run --quiet core:version)." - exit 0 - fi - for t in $tags; do - mise run core:bump -- "$t" - done - - - name: Report failure - if: failure() - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: mise run core:report-failure -- core-watch "$RUN_URL" + uses: pimlock/workflows/.github/workflows/core-watch.yml@2e0e58cd14e4a3484d0458a40466c77ef48b5f4f # feat/core-sync/pimlock + with: + tag: ${{ inputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0191316..c0aad60 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,40 +28,13 @@ jobs: name: tag # Only main releases, also when the workflow is started by hand. if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - timeout-minutes: 10 + # Shared with the other SDKs; same commit as core-watch.yml and core-drift.yml. + uses: pimlock/workflows/.github/workflows/release-tag.yml@2e0e58cd14e4a3484d0458a40466c77ef48b5f4f # feat/core-sync/pimlock permissions: contents: write # create the tag and the draft release pull-requests: read # read the merged PR's body for the release notes - outputs: - tag: ${{ steps.tag.outputs.tag }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - id: tag - name: Tag the release and create a draft - env: - GH_TOKEN: ${{ github.token }} - EVENT: ${{ github.event_name }} - INPUT_SHA: ${{ inputs.sha }} - BEFORE: ${{ github.event.before }} - run: | - set -euo pipefail - if [ "$EVENT" = workflow_dispatch ]; then - sha=${INPUT_SHA:-$GITHUB_SHA} - git merge-base --is-ancestor "$sha" "$GITHUB_SHA" || { - echo "::error::$sha is not on main" - exit 1 - } - scripts/release/tag-release.sh --force --sha "$sha" >>"$GITHUB_OUTPUT" - else - # Compare against the commit before the push, so a pin change - # that isn't the pushed tip still counts. - scripts/release/tag-release.sh --since "$BEFORE" >>"$GITHUB_OUTPUT" - fi + with: + sha: ${{ inputs.sha }} build: name: build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2cc7e63..fb6285e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,7 @@ Merging a pull request that moves the pin to a stable core tag releases the SDK. changed to a stable tag. Then it: 1. tags `vX.Y.Z` at the merged commit and creates a draft GitHub release, with - the pull request's description as notes (`scripts/release/tag-release.sh`); + the pull request's description as notes (`mise run release:tag`, shared through ai-openshell/workflows); 2. builds the wheel and sdist at version `X.Y.Z`, verifies them, and writes their checksums; 3. publishes to PyPI from the `pypi` environment with trusted publishing; diff --git a/mise.toml b/mise.toml index b30b961..6c72df3 100644 --- a/mise.toml +++ b/mise.toml @@ -17,8 +17,10 @@ buf = "1.72.0" uv = "0.10.12" [env] -# The single core pin: a release tag. scripts/core-sync/ requires a tag. +# The single core pin: a release tag. The shared core-sync scripts require a tag. OPENSHELL_CORE_REF = "v0.1.9" +# The committed generated bindings, for the core-sync summaries. +OPENSHELL_GENERATED_PATHS = "src/openshell/_proto" # The package version for builds. CI and local builds use 0.0.0; the release # workflow sets OPENSHELL_SDK_VERSION to the release's version. SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENSHELL = "{{ get_env(name='OPENSHELL_SDK_VERSION', default='0.0.0') }}" @@ -147,22 +149,40 @@ done description = "Run the full CI pipeline" depends = ["check", "test", "build"] +# --- Core sync -------------------------------------------------------------- +# The scripts are shared through ai-openshell/workflows (core-sync/). +# scripts/core-sync.sh runs them at the commit the workflows call (the `uses:` +# line in core-drift.yml is the pin). They call the tasks below this block +# (core-sync/README.md there has the contract). + [tasks."core:tags"] description = "List core release tags newer than the pin (newest pre-release only, --all for every tag)" -run = "scripts/core-sync/core-tags.sh" +run = "scripts/core-sync.sh core-tags.sh" [tasks."core:bump"] description = "Open the update proposal for a core tag: mise run core:bump -- [--dry-run]" -run = "scripts/core-sync/open-bump-pr.sh" +run = "scripts/core-sync.sh open-bump-pr.sh" [tasks."core:drift"] description = "Compare core main with the pin: mise run core:drift -- --out " -run = "scripts/core-sync/drift-check.sh" +run = "scripts/core-sync.sh drift-check.sh" [tasks."core:drift-issue"] description = "Create, update, or close the core main drift issue from a report: mise run core:drift-issue -- [--dry-run]" -run = "scripts/core-sync/update-drift-issue.sh" +run = "scripts/core-sync.sh update-drift-issue.sh" [tasks."core:report-failure"] description = "Record a failed automation run in the drift issue: mise run core:report-failure -- [--dry-run]" -run = "scripts/core-sync/report-failure.sh" +run = "scripts/core-sync.sh report-failure.sh" + +[tasks."release:tag"] +description = "Tag and draft the release when a stable pin lands on main: mise run release:tag -- [--dry-run] [--force] [--sha ] [--since ]" +run = "scripts/core-sync.sh tag-release.sh" + +[tasks."proto:buf"] +description = "Run buf: mise run proto:buf -- " +run = "buf" + +[tasks."core:uncovered-rpcs"] +description = "List the RPCs in src/openshell/_proto that the SDK doesn't call, one Service.Method per line" +run = "scripts/uncovered-rpcs.sh" diff --git a/scripts/README.md b/scripts/README.md index e0757b6..b5866f8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,26 +2,23 @@ The workflows under `.github/workflows/` run these scripts, most through a mise task, so each job can be run locally first. Scripts that write to GitHub take `--dry-run` and print what they would do instead. -## `core-sync/`: keeping up with core +## Core sync and release tagging (shared) -Move the core pin (`OPENSHELL_CORE_REF` in `mise.toml`) to new core releases, and watch core `main` for changes that will break the SDK. +The core-sync scripts and the `core-watch` and `core-drift` workflows are shared with the other SDKs through [ai-openshell/workflows](https://github.com/ai-openshell/workflows/tree/main/core-sync), which documents them and the contract this repository implements. The workflows (`core-watch.yml`, `core-drift.yml`, and the tag job of `release.yml`) call the shared workflows at one commit, and the shared workflows run that commit's scripts. Locally, `core-sync.sh` runs the scripts of the commit in `core-drift.yml`'s `uses:` line, downloaded once into `~/.cache/openshell-core-sync/`. -The workflows use a GitHub App token from the `sdk-bot` environment (secrets `SDK_BOT_CLIENT_ID` and `SDK_BOT_PRIVATE_KEY`), which only `main` can deploy to. They never use `GITHUB_TOKEN`, because pushes and PRs made with it don't trigger CI. +| mise task | Runs | +|-----------|------| +| `core:tags`, `core:bump`, `core:drift`, `core:drift-issue`, `core:report-failure`, `release:tag` | The shared scripts, through `core-sync.sh`. | +| `proto:fetch`, `proto:gen`, `proto:buf`, `check`, `test` | This repository's parts of the contract. | +| `core:uncovered-rpcs` | `uncovered-rpcs.sh`: RPCs in `src/openshell/_proto` the SDK doesn't call, for the drift report. | -| File | mise task | Purpose | -|------|-----------|---------| -| `core-tags.sh [--all]` | `core:tags` | List NVIDIA/OpenShell release tags newer than the pin: every stable tag, plus the newest pre-release when it is newer than all of them. | -| `open-bump-pr.sh [--dry-run]` | `core:bump` | Open the update proposal for one core tag: branch `core/`, pin change, regenerated bindings, refreshed copies of core files, proto diff in the body, labels `core-bump` plus `stable` or `prerelease`. Closes older pre-release proposals. | -| `bump-pr-body.md` | | PR body template for `open-bump-pr.sh`. | -| `drift-check.sh --out ` | `core:drift` | Compare core `main` with the pin: `buf breaking`, regenerate and run the unit tests, list the RPCs the SDK doesn't call. Writes the markdown report. | -| `update-drift-issue.sh [--dry-run]` | `core:drift-issue` | Create, update, or close the single `Core main drift` issue from a report. | -| `report-failure.sh [--dry-run]` | `core:report-failure` | Append a dated failure line to the drift issue. Called from `if: failure()` steps. | -| `drift-issue.sh` | | Shared helpers for the two drift issue scripts (sourced, not run). | - -| Workflow | Runs (mise tasks) | +| Workflow | Runs | |----------|------| | `core-watch.yml` (hourly) | `core:tags`, then `core:bump` per tag | | `core-drift.yml` (nightly) | `core:drift`, then `core:drift-issue` | +| `release.yml` (push to `main`) | `release:tag`; when it prints a tag, builds and publishes the release | + +The shared workflows use a GitHub App token from the `sdk-bot` environment (secrets `SDK_BOT_CLIENT_ID` and `SDK_BOT_PRIVATE_KEY`), which only `main` can deploy to. Bump PRs run E2E against their own pin, because `e2e.yml` defaults to `OPENSHELL_CORE_REF` on the PR branch. @@ -40,5 +37,5 @@ Bump PRs run E2E against their own pin, because `e2e.yml` defaults to `OPENSHELL |------|-----------|---------| | `generate_python_proto.py` | `proto:gen` | Generate `src/openshell/_proto` from `proto/`. | | `verify-python-wheel.py` | `build` | Check the wheel's contents and version. | -| `vendored-core-files.txt` | `vendored:check` | Files copied from core. `vendored:check` compares them with core at the pin; `core-sync/open-bump-pr.sh` refreshes them. | +| `vendored-core-files.txt` | `vendored:check` | Files copied from core. `vendored:check` compares them with core at the pin; `core:bump` refreshes them. | | `fetch-core.sh ...` | `vendored:check` | Copy files from core at a ref. | diff --git a/scripts/core-sync.sh b/scripts/core-sync.sh new file mode 100755 index 0000000..dc72b81 --- /dev/null +++ b/scripts/core-sync.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +# core-sync.sh: run a shared core-sync script from ai-openshell/workflows. +# SDK repositories keep a copy of this file as scripts/core-sync.sh and point +# their core-sync mise tasks at it. +# +# Usage: scripts/core-sync.sh